summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMark Hatle <mark.hatle@amd.com>2025-03-20 14:45:23 -0600
committerMark Hatle <mark.hatle@amd.com>2025-03-30 14:16:15 -0600
commitaac160dbfa1e4efd6e8239bfbc1d7fe97364d87c (patch)
tree9ae86099e835c740fb51baf2f8cd8b1f223840f3
parent4a95d4cdccb17a5630787ed7e5097b3c3b42891a (diff)
downloadmeta-xilinx-aac160dbfa1e4efd6e8239bfbc1d7fe97364d87c.tar.gz
meta-xilinx-core: Bring in a copy of wic from poky
Import a copy of wic and lib/oe as of poky commit: 8f74fa4073d4b2ba8e0d9559aa654f3cafcf373a Signed-off-by: Mark Hatle <mark.hatle@amd.com>
-rw-r--r--meta-xilinx-core/lib/oe/__init__.py12
-rw-r--r--meta-xilinx-core/lib/oe/buildcfg.py79
-rw-r--r--meta-xilinx-core/lib/oe/buildhistory_analysis.py723
-rw-r--r--meta-xilinx-core/lib/oe/cachedpath.py237
-rw-r--r--meta-xilinx-core/lib/oe/classextend.py159
-rw-r--r--meta-xilinx-core/lib/oe/classutils.py49
-rw-r--r--meta-xilinx-core/lib/oe/copy_buildsystem.py293
-rw-r--r--meta-xilinx-core/lib/oe/cve_check.py245
-rw-r--r--meta-xilinx-core/lib/oe/data.py53
-rw-r--r--meta-xilinx-core/lib/oe/distro_check.py314
-rw-r--r--meta-xilinx-core/lib/oe/elf.py145
-rw-r--r--meta-xilinx-core/lib/oe/go.py34
-rw-r--r--meta-xilinx-core/lib/oe/gpg_sign.py160
-rw-r--r--meta-xilinx-core/lib/oe/license.py261
-rw-r--r--meta-xilinx-core/lib/oe/lsb.py123
-rw-r--r--meta-xilinx-core/lib/oe/maketype.py107
-rw-r--r--meta-xilinx-core/lib/oe/manifest.py206
-rw-r--r--meta-xilinx-core/lib/oe/npm_registry.py175
-rw-r--r--meta-xilinx-core/lib/oe/overlayfs.py54
-rw-r--r--meta-xilinx-core/lib/oe/package.py2065
-rw-r--r--meta-xilinx-core/lib/oe/packagedata.py366
-rw-r--r--meta-xilinx-core/lib/oe/packagegroup.py36
-rw-r--r--meta-xilinx-core/lib/oe/patch.py1001
-rw-r--r--meta-xilinx-core/lib/oe/path.py349
-rw-r--r--meta-xilinx-core/lib/oe/prservice.py127
-rw-r--r--meta-xilinx-core/lib/oe/qa.py238
-rw-r--r--meta-xilinx-core/lib/oe/recipeutils.py1185
-rw-r--r--meta-xilinx-core/lib/oe/reproducible.py197
-rw-r--r--meta-xilinx-core/lib/oe/rootfs.py438
-rw-r--r--meta-xilinx-core/lib/oe/rust.py13
-rw-r--r--meta-xilinx-core/lib/oe/sbom.py120
-rw-r--r--meta-xilinx-core/lib/oe/sdk.py160
-rw-r--r--meta-xilinx-core/lib/oe/spdx.py357
-rw-r--r--meta-xilinx-core/lib/oe/sstatesig.py691
-rw-r--r--meta-xilinx-core/lib/oe/terminal.py332
-rw-r--r--meta-xilinx-core/lib/oe/types.py188
-rw-r--r--meta-xilinx-core/lib/oe/useradd.py71
-rw-r--r--meta-xilinx-core/lib/oe/utils.py529
-rw-r--r--meta-xilinx-core/scripts/lib/scriptpath.py32
-rw-r--r--meta-xilinx-core/scripts/lib/wic/__init__.py10
-rw-r--r--meta-xilinx-core/scripts/lib/wic/canned-wks/common.wks.inc3
-rw-r--r--meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg27
-rw-r--r--meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-bootloader-config.wks8
-rw-r--r--meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-gpt.wks10
-rw-r--r--meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks23
-rw-r--r--meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk.wks8
-rw-r--r--meta-xilinx-core/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in3
-rw-r--r--meta-xilinx-core/scripts/lib/wic/canned-wks/mkefidisk.wks11
-rw-r--r--meta-xilinx-core/scripts/lib/wic/canned-wks/mkhybridiso.wks7
-rw-r--r--meta-xilinx-core/scripts/lib/wic/canned-wks/qemuloongarch.wks3
-rw-r--r--meta-xilinx-core/scripts/lib/wic/canned-wks/qemuriscv.wks3
-rw-r--r--meta-xilinx-core/scripts/lib/wic/canned-wks/qemux86-directdisk.wks8
-rw-r--r--meta-xilinx-core/scripts/lib/wic/canned-wks/sdimage-bootpart.wks6
-rw-r--r--meta-xilinx-core/scripts/lib/wic/canned-wks/systemd-bootdisk.wks11
-rw-r--r--meta-xilinx-core/scripts/lib/wic/engine.py628
-rw-r--r--meta-xilinx-core/scripts/lib/wic/filemap.py583
-rw-r--r--meta-xilinx-core/scripts/lib/wic/help.py1148
-rw-r--r--meta-xilinx-core/scripts/lib/wic/ksparser.py298
-rw-r--r--meta-xilinx-core/scripts/lib/wic/misc.py266
-rw-r--r--meta-xilinx-core/scripts/lib/wic/partition.py551
-rw-r--r--meta-xilinx-core/scripts/lib/wic/pluginbase.py144
-rw-r--r--meta-xilinx-core/scripts/lib/wic/plugins/imager/direct.py694
-rw-r--r--meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-biosplusefi.py213
-rw-r--r--meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-efi.py507
-rw-r--r--meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-partition.py197
-rw-r--r--meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-pcbios.py209
-rw-r--r--meta-xilinx-core/scripts/lib/wic/plugins/source/empty.py89
-rw-r--r--meta-xilinx-core/scripts/lib/wic/plugins/source/isoimage-isohybrid.py463
-rw-r--r--meta-xilinx-core/scripts/lib/wic/plugins/source/rawcopy.py115
-rw-r--r--meta-xilinx-core/scripts/lib/wic/plugins/source/rootfs.py236
-rwxr-xr-xmeta-xilinx-core/scripts/wic551
71 files changed, 18957 insertions, 0 deletions
diff --git a/meta-xilinx-core/lib/oe/__init__.py b/meta-xilinx-core/lib/oe/__init__.py
new file mode 100644
index 00000000..6eb536ad
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/__init__.py
@@ -0,0 +1,12 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7from pkgutil import extend_path
8__path__ = extend_path(__path__, __name__)
9
10BBIMPORTS = ["data", "path", "utils", "types", "package", "packagedata", \
11 "packagegroup", "sstatesig", "lsb", "cachedpath", "license", \
12 "qa", "reproducible", "rust", "buildcfg", "go"]
diff --git a/meta-xilinx-core/lib/oe/buildcfg.py b/meta-xilinx-core/lib/oe/buildcfg.py
new file mode 100644
index 00000000..27b059b8
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/buildcfg.py
@@ -0,0 +1,79 @@
1
2import os
3import subprocess
4import bb.process
5
6def detect_revision(d):
7 path = get_scmbasepath(d)
8 return get_metadata_git_revision(path)
9
10def detect_branch(d):
11 path = get_scmbasepath(d)
12 return get_metadata_git_branch(path)
13
14def get_scmbasepath(d):
15 return os.path.join(d.getVar('COREBASE'), 'meta')
16
17def get_metadata_git_branch(path):
18 try:
19 rev, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', cwd=path)
20 except bb.process.ExecutionError:
21 rev = '<unknown>'
22 return rev.strip()
23
24def get_metadata_git_revision(path):
25 try:
26 rev, _ = bb.process.run('git rev-parse HEAD', cwd=path)
27 except bb.process.ExecutionError:
28 rev = '<unknown>'
29 return rev.strip()
30
31def get_metadata_git_toplevel(path):
32 try:
33 toplevel, _ = bb.process.run('git rev-parse --show-toplevel', cwd=path)
34 except bb.process.ExecutionError:
35 return ""
36 return toplevel.strip()
37
38def get_metadata_git_remotes(path):
39 try:
40 remotes_list, _ = bb.process.run('git remote', cwd=path)
41 remotes = remotes_list.split()
42 except bb.process.ExecutionError:
43 remotes = []
44 return remotes
45
46def get_metadata_git_remote_url(path, remote):
47 try:
48 uri, _ = bb.process.run('git remote get-url {remote}'.format(remote=remote), cwd=path)
49 except bb.process.ExecutionError:
50 return ""
51 return uri.strip()
52
53def get_metadata_git_describe(path):
54 try:
55 describe, _ = bb.process.run('git describe --tags', cwd=path)
56 except bb.process.ExecutionError:
57 return ""
58 return describe.strip()
59
60def is_layer_modified(path):
61 try:
62 subprocess.check_output("""cd %s; export PSEUDO_UNLOAD=1; set -e;
63 git diff --quiet --no-ext-diff
64 git diff --quiet --no-ext-diff --cached""" % path,
65 shell=True,
66 stderr=subprocess.STDOUT)
67 return ""
68 except subprocess.CalledProcessError as ex:
69 # Silently treat errors as "modified", without checking for the
70 # (expected) return code 1 in a modified git repo. For example, we get
71 # output and a 129 return code when a layer isn't a git repo at all.
72 return " -- modified"
73
74def get_layer_revisions(d):
75 layers = (d.getVar("BBLAYERS") or "").split()
76 revisions = []
77 for i in layers:
78 revisions.append((i, os.path.basename(i), get_metadata_git_branch(i).strip(), get_metadata_git_revision(i), is_layer_modified(i)))
79 return revisions
diff --git a/meta-xilinx-core/lib/oe/buildhistory_analysis.py b/meta-xilinx-core/lib/oe/buildhistory_analysis.py
new file mode 100644
index 00000000..4edad015
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/buildhistory_analysis.py
@@ -0,0 +1,723 @@
1# Report significant differences in the buildhistory repository since a specific revision
2#
3# Copyright (C) 2012-2013, 2016-2017 Intel Corporation
4# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
5#
6# SPDX-License-Identifier: GPL-2.0-only
7#
8# Note: requires GitPython 0.3.1+
9#
10# You can use this from the command line by running scripts/buildhistory-diff
11#
12
13import sys
14import os.path
15import difflib
16import git
17import re
18import shlex
19import hashlib
20import collections
21import bb.utils
22import bb.tinfoil
23
24
25# How to display fields
26list_fields = ['DEPENDS', 'RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS', 'FILES', 'FILELIST', 'USER_CLASSES', 'IMAGE_CLASSES', 'IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS', 'PACKAGE_EXCLUDE']
27list_order_fields = ['PACKAGES']
28defaultval_map = {'PKG': 'PKG', 'PKGE': 'PE', 'PKGV': 'PV', 'PKGR': 'PR'}
29numeric_fields = ['PKGSIZE', 'IMAGESIZE']
30# Fields to monitor
31monitor_fields = ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RREPLACES', 'RCONFLICTS', 'PACKAGES', 'FILELIST', 'PKGSIZE', 'IMAGESIZE', 'PKG']
32ver_monitor_fields = ['PKGE', 'PKGV', 'PKGR']
33# Percentage change to alert for numeric fields
34monitor_numeric_threshold = 10
35# Image files to monitor (note that image-info.txt is handled separately)
36img_monitor_files = ['installed-package-names.txt', 'files-in-image.txt']
37
38colours = {
39 'colour_default': '',
40 'colour_add': '',
41 'colour_remove': '',
42}
43
44def init_colours(use_colours):
45 global colours
46 if use_colours:
47 colours = {
48 'colour_default': '\033[0m',
49 'colour_add': '\033[1;32m',
50 'colour_remove': '\033[1;31m',
51 }
52 else:
53 colours = {
54 'colour_default': '',
55 'colour_add': '',
56 'colour_remove': '',
57 }
58
59class ChangeRecord:
60 def __init__(self, path, fieldname, oldvalue, newvalue, monitored):
61 self.path = path
62 self.fieldname = fieldname
63 self.oldvalue = oldvalue
64 self.newvalue = newvalue
65 self.monitored = monitored
66 self.filechanges = None
67
68 def __str__(self):
69 return self._str_internal(True)
70
71 def _str_internal(self, outer):
72 if outer:
73 if '/image-files/' in self.path:
74 prefix = '%s: ' % self.path.split('/image-files/')[0]
75 else:
76 prefix = '%s: ' % self.path
77 else:
78 prefix = ''
79
80 def pkglist_combine(depver):
81 pkglist = []
82 for k,v in depver.items():
83 if v:
84 pkglist.append("%s (%s)" % (k,v))
85 else:
86 pkglist.append(k)
87 return pkglist
88
89 def detect_renamed_dirs(aitems, bitems):
90 adirs = set(map(os.path.dirname, aitems))
91 bdirs = set(map(os.path.dirname, bitems))
92 files_ab = [(name, sorted(os.path.basename(item) for item in aitems if os.path.dirname(item) == name)) \
93 for name in adirs - bdirs]
94 files_ba = [(name, sorted(os.path.basename(item) for item in bitems if os.path.dirname(item) == name)) \
95 for name in bdirs - adirs]
96 renamed_dirs = []
97 for dir1, files1 in files_ab:
98 rename = False
99 for dir2, files2 in files_ba:
100 if files1 == files2 and not rename:
101 renamed_dirs.append((dir1,dir2))
102 # Make sure that we don't use this (dir, files) pair again.
103 files_ba.remove((dir2,files2))
104 # If a dir has already been found to have a rename, stop and go no further.
105 rename = True
106
107 # remove files that belong to renamed dirs from aitems and bitems
108 for dir1, dir2 in renamed_dirs:
109 aitems = [item for item in aitems if os.path.dirname(item) not in (dir1, dir2)]
110 bitems = [item for item in bitems if os.path.dirname(item) not in (dir1, dir2)]
111 return renamed_dirs, aitems, bitems
112
113 if self.fieldname in list_fields or self.fieldname in list_order_fields:
114 renamed_dirs = []
115 changed_order = False
116 if self.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']:
117 (depvera, depverb) = compare_pkg_lists(self.oldvalue, self.newvalue)
118 aitems = pkglist_combine(depvera)
119 bitems = pkglist_combine(depverb)
120 else:
121 if self.fieldname == 'FILELIST':
122 aitems = shlex.split(self.oldvalue)
123 bitems = shlex.split(self.newvalue)
124 renamed_dirs, aitems, bitems = detect_renamed_dirs(aitems, bitems)
125 else:
126 aitems = self.oldvalue.split()
127 bitems = self.newvalue.split()
128
129 removed = list(set(aitems) - set(bitems))
130 added = list(set(bitems) - set(aitems))
131
132 if not removed and not added and self.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']:
133 depvera = bb.utils.explode_dep_versions2(self.oldvalue, sort=False)
134 depverb = bb.utils.explode_dep_versions2(self.newvalue, sort=False)
135 for i, j in zip(depvera.items(), depverb.items()):
136 if i[0] != j[0]:
137 changed_order = True
138 break
139
140 lines = []
141 if renamed_dirs:
142 for dfrom, dto in renamed_dirs:
143 lines.append('directory renamed {colour_remove}{}{colour_default} -> {colour_add}{}{colour_default}'.format(dfrom, dto, **colours))
144 if removed or added:
145 if removed and not bitems:
146 lines.append('removed all items "{colour_remove}{}{colour_default}"'.format(' '.join(removed), **colours))
147 else:
148 if removed:
149 lines.append('removed "{colour_remove}{value}{colour_default}"'.format(value=' '.join(removed), **colours))
150 if added:
151 lines.append('added "{colour_add}{value}{colour_default}"'.format(value=' '.join(added), **colours))
152 else:
153 lines.append('changed order')
154
155 if not (removed or added or changed_order):
156 out = ''
157 else:
158 out = '%s: %s' % (self.fieldname, ', '.join(lines))
159
160 elif self.fieldname in numeric_fields:
161 aval = int(self.oldvalue or 0)
162 bval = int(self.newvalue or 0)
163 if aval != 0:
164 percentchg = ((bval - aval) / float(aval)) * 100
165 else:
166 percentchg = 100
167 out = '{} changed from {colour_remove}{}{colour_default} to {colour_add}{}{colour_default} ({}{:.0f}%)'.format(self.fieldname, self.oldvalue or "''", self.newvalue or "''", '+' if percentchg > 0 else '', percentchg, **colours)
168 elif self.fieldname in defaultval_map:
169 out = '{} changed from {colour_remove}{}{colour_default} to {colour_add}{}{colour_default}'.format(self.fieldname, self.oldvalue, self.newvalue, **colours)
170 if self.fieldname == 'PKG' and '[default]' in self.newvalue:
171 out += ' - may indicate debian renaming failure'
172 elif self.fieldname in ['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm']:
173 if self.oldvalue and self.newvalue:
174 out = '%s changed:\n ' % self.fieldname
175 elif self.newvalue:
176 out = '%s added:\n ' % self.fieldname
177 elif self.oldvalue:
178 out = '%s cleared:\n ' % self.fieldname
179 alines = self.oldvalue.splitlines()
180 blines = self.newvalue.splitlines()
181 diff = difflib.unified_diff(alines, blines, self.fieldname, self.fieldname, lineterm='')
182 out += '\n '.join(list(diff)[2:])
183 out += '\n --'
184 elif self.fieldname in img_monitor_files or '/image-files/' in self.path or self.fieldname == "sysroot":
185 if self.filechanges or (self.oldvalue and self.newvalue):
186 fieldname = self.fieldname
187 if '/image-files/' in self.path:
188 fieldname = os.path.join('/' + self.path.split('/image-files/')[1], self.fieldname)
189 out = 'Changes to %s:\n ' % fieldname
190 else:
191 if outer:
192 prefix = 'Changes to %s ' % self.path
193 out = '(%s):\n ' % self.fieldname
194 if self.filechanges:
195 out += '\n '.join(['%s' % i for i in self.filechanges])
196 else:
197 alines = self.oldvalue.splitlines()
198 blines = self.newvalue.splitlines()
199 diff = difflib.unified_diff(alines, blines, fieldname, fieldname, lineterm='')
200 out += '\n '.join(list(diff))
201 out += '\n --'
202 else:
203 out = ''
204 else:
205 out = '{} changed from "{colour_remove}{}{colour_default}" to "{colour_add}{}{colour_default}"'.format(self.fieldname, self.oldvalue, self.newvalue, **colours)
206
207 return '%s%s' % (prefix, out) if out else ''
208
209class FileChange:
210 changetype_add = 'A'
211 changetype_remove = 'R'
212 changetype_type = 'T'
213 changetype_perms = 'P'
214 changetype_ownergroup = 'O'
215 changetype_link = 'L'
216 changetype_move = 'M'
217
218 def __init__(self, path, changetype, oldvalue = None, newvalue = None):
219 self.path = path
220 self.changetype = changetype
221 self.oldvalue = oldvalue
222 self.newvalue = newvalue
223
224 def _ftype_str(self, ftype):
225 if ftype == '-':
226 return 'file'
227 elif ftype == 'd':
228 return 'directory'
229 elif ftype == 'l':
230 return 'symlink'
231 elif ftype == 'c':
232 return 'char device'
233 elif ftype == 'b':
234 return 'block device'
235 elif ftype == 'p':
236 return 'fifo'
237 elif ftype == 's':
238 return 'socket'
239 else:
240 return 'unknown (%s)' % ftype
241
242 def __str__(self):
243 if self.changetype == self.changetype_add:
244 return '%s was added' % self.path
245 elif self.changetype == self.changetype_remove:
246 return '%s was removed' % self.path
247 elif self.changetype == self.changetype_type:
248 return '%s changed type from %s to %s' % (self.path, self._ftype_str(self.oldvalue), self._ftype_str(self.newvalue))
249 elif self.changetype == self.changetype_perms:
250 return '%s changed permissions from %s to %s' % (self.path, self.oldvalue, self.newvalue)
251 elif self.changetype == self.changetype_ownergroup:
252 return '%s changed owner/group from %s to %s' % (self.path, self.oldvalue, self.newvalue)
253 elif self.changetype == self.changetype_link:
254 return '%s changed symlink target from %s to %s' % (self.path, self.oldvalue, self.newvalue)
255 elif self.changetype == self.changetype_move:
256 return '%s moved to %s' % (self.path, self.oldvalue)
257 else:
258 return '%s changed (unknown)' % self.path
259
260def blob_to_dict(blob):
261 alines = [line for line in blob.data_stream.read().decode('utf-8').splitlines()]
262 adict = {}
263 for line in alines:
264 splitv = [i.strip() for i in line.split('=',1)]
265 if len(splitv) > 1:
266 adict[splitv[0]] = splitv[1]
267 return adict
268
269
270def file_list_to_dict(lines):
271 adict = {}
272 for line in lines:
273 # Leave the last few fields intact so we handle file names containing spaces
274 splitv = line.split(None,4)
275 # Grab the path and remove the leading .
276 path = splitv[4][1:].strip()
277 # Handle symlinks
278 if(' -> ' in path):
279 target = path.split(' -> ')[1]
280 path = path.split(' -> ')[0]
281 adict[path] = splitv[0:3] + [target]
282 else:
283 adict[path] = splitv[0:3]
284 return adict
285
286numeric_removal = str.maketrans('0123456789', 'XXXXXXXXXX')
287
288def compare_file_lists(alines, blines, compare_ownership=True):
289 adict = file_list_to_dict(alines)
290 bdict = file_list_to_dict(blines)
291 filechanges = []
292 additions = []
293 removals = []
294 for path, splitv in adict.items():
295 newsplitv = bdict.pop(path, None)
296 if newsplitv:
297 # Check type
298 oldvalue = splitv[0][0]
299 newvalue = newsplitv[0][0]
300 if oldvalue != newvalue:
301 filechanges.append(FileChange(path, FileChange.changetype_type, oldvalue, newvalue))
302
303 # Check permissions
304 oldvalue = splitv[0][1:]
305 newvalue = newsplitv[0][1:]
306 if oldvalue != newvalue:
307 filechanges.append(FileChange(path, FileChange.changetype_perms, oldvalue, newvalue))
308
309 if compare_ownership:
310 # Check owner/group
311 oldvalue = '%s/%s' % (splitv[1], splitv[2])
312 newvalue = '%s/%s' % (newsplitv[1], newsplitv[2])
313 if oldvalue != newvalue:
314 filechanges.append(FileChange(path, FileChange.changetype_ownergroup, oldvalue, newvalue))
315
316 # Check symlink target
317 if newsplitv[0][0] == 'l':
318 if len(splitv) > 3:
319 oldvalue = splitv[3]
320 else:
321 oldvalue = None
322 newvalue = newsplitv[3]
323 if oldvalue != newvalue:
324 filechanges.append(FileChange(path, FileChange.changetype_link, oldvalue, newvalue))
325 else:
326 removals.append(path)
327
328 # Whatever is left over has been added
329 for path in bdict:
330 additions.append(path)
331
332 # Rather than print additions and removals, its nicer to print file 'moves'
333 # where names or paths are similar.
334 revmap_remove = {}
335 for removal in removals:
336 translated = removal.translate(numeric_removal)
337 if translated not in revmap_remove:
338 revmap_remove[translated] = []
339 revmap_remove[translated].append(removal)
340
341 #
342 # We want to detect renames of large trees of files like
343 # /lib/modules/5.4.40-yocto-standard to /lib/modules/5.4.43-yocto-standard
344 #
345 renames = {}
346 for addition in additions.copy():
347 if addition not in additions:
348 continue
349 translated = addition.translate(numeric_removal)
350 if translated in revmap_remove:
351 if len(revmap_remove[translated]) != 1:
352 continue
353 removal = revmap_remove[translated][0]
354 commondir = addition.split("/")
355 commondir2 = removal.split("/")
356 idx = None
357 for i in range(len(commondir)):
358 if commondir[i] != commondir2[i]:
359 idx = i
360 break
361 commondir = "/".join(commondir[:i+1])
362 commondir2 = "/".join(commondir2[:i+1])
363 # If the common parent is in one dict and not the other its likely a rename
364 # so iterate through those files and process as such
365 if commondir2 not in bdict and commondir not in adict:
366 if commondir not in renames:
367 renames[commondir] = commondir2
368 for addition2 in additions.copy():
369 if addition2.startswith(commondir):
370 removal2 = addition2.replace(commondir, commondir2)
371 if removal2 in removals:
372 additions.remove(addition2)
373 removals.remove(removal2)
374 continue
375 filechanges.append(FileChange(removal, FileChange.changetype_move, addition))
376 if addition in additions:
377 additions.remove(addition)
378 if removal in removals:
379 removals.remove(removal)
380 for rename in renames:
381 filechanges.append(FileChange(renames[rename], FileChange.changetype_move, rename))
382
383 for addition in additions:
384 filechanges.append(FileChange(addition, FileChange.changetype_add))
385 for removal in removals:
386 filechanges.append(FileChange(removal, FileChange.changetype_remove))
387
388 return filechanges
389
390
391def compare_lists(alines, blines):
392 removed = list(set(alines) - set(blines))
393 added = list(set(blines) - set(alines))
394
395 filechanges = []
396 for pkg in removed:
397 filechanges.append(FileChange(pkg, FileChange.changetype_remove))
398 for pkg in added:
399 filechanges.append(FileChange(pkg, FileChange.changetype_add))
400
401 return filechanges
402
403
404def compare_pkg_lists(astr, bstr):
405 depvera = bb.utils.explode_dep_versions2(astr)
406 depverb = bb.utils.explode_dep_versions2(bstr)
407
408 # Strip out changes where the version has increased
409 remove = []
410 for k in depvera:
411 if k in depverb:
412 dva = depvera[k]
413 dvb = depverb[k]
414 if dva and dvb and len(dva) == len(dvb):
415 # Since length is the same, sort so that prefixes (e.g. >=) will line up
416 dva.sort()
417 dvb.sort()
418 removeit = True
419 for dvai, dvbi in zip(dva, dvb):
420 if dvai != dvbi:
421 aiprefix = dvai.split(' ')[0]
422 biprefix = dvbi.split(' ')[0]
423 if aiprefix == biprefix and aiprefix in ['>=', '=']:
424 if bb.utils.vercmp(bb.utils.split_version(dvai), bb.utils.split_version(dvbi)) > 0:
425 removeit = False
426 break
427 else:
428 removeit = False
429 break
430 if removeit:
431 remove.append(k)
432
433 for k in remove:
434 depvera.pop(k)
435 depverb.pop(k)
436
437 return (depvera, depverb)
438
439
440def compare_dict_blobs(path, ablob, bblob, report_all, report_ver):
441 adict = blob_to_dict(ablob)
442 bdict = blob_to_dict(bblob)
443
444 pkgname = os.path.basename(path)
445
446 defaultvals = {}
447 defaultvals['PKG'] = pkgname
448 defaultvals['PKGE'] = '0'
449
450 changes = []
451 keys = list(set(adict.keys()) | set(bdict.keys()) | set(defaultval_map.keys()))
452 for key in keys:
453 astr = adict.get(key, '')
454 bstr = bdict.get(key, '')
455 if key in ver_monitor_fields:
456 monitored = report_ver or astr or bstr
457 else:
458 monitored = key in monitor_fields
459 mapped_key = defaultval_map.get(key, '')
460 if mapped_key:
461 if not astr:
462 astr = '%s [default]' % adict.get(mapped_key, defaultvals.get(key, ''))
463 if not bstr:
464 bstr = '%s [default]' % bdict.get(mapped_key, defaultvals.get(key, ''))
465
466 if astr != bstr:
467 if (not report_all) and key in numeric_fields:
468 aval = int(astr or 0)
469 bval = int(bstr or 0)
470 if aval != 0:
471 percentchg = ((bval - aval) / float(aval)) * 100
472 else:
473 percentchg = 100
474 if abs(percentchg) < monitor_numeric_threshold:
475 continue
476 elif (not report_all) and key in list_fields:
477 if key == "FILELIST" and (path.endswith("-dbg") or path.endswith("-src")) and bstr.strip() != '':
478 continue
479 if key in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']:
480 (depvera, depverb) = compare_pkg_lists(astr, bstr)
481 if depvera == depverb:
482 continue
483 if key == 'FILELIST':
484 alist = shlex.split(astr)
485 blist = shlex.split(bstr)
486 else:
487 alist = astr.split()
488 blist = bstr.split()
489 alist.sort()
490 blist.sort()
491 # We don't care about the removal of self-dependencies
492 if pkgname in alist and not pkgname in blist:
493 alist.remove(pkgname)
494 if ' '.join(alist) == ' '.join(blist):
495 continue
496
497 if key == 'PKGR' and not report_all:
498 vers = []
499 # strip leading 'r' and dots
500 for ver in (astr.split()[0], bstr.split()[0]):
501 if ver.startswith('r'):
502 ver = ver[1:]
503 vers.append(ver.replace('.', ''))
504 maxlen = max(len(vers[0]), len(vers[1]))
505 try:
506 # pad with '0' and convert to int
507 vers = [int(ver.ljust(maxlen, '0')) for ver in vers]
508 except ValueError:
509 pass
510 else:
511 # skip decrements and increments
512 if abs(vers[0] - vers[1]) == 1:
513 continue
514
515 chg = ChangeRecord(path, key, astr, bstr, monitored)
516 changes.append(chg)
517 return changes
518
519
520def compare_siglists(a_blob, b_blob, taskdiff=False):
521 # FIXME collapse down a recipe's tasks?
522 alines = a_blob.data_stream.read().decode('utf-8').splitlines()
523 blines = b_blob.data_stream.read().decode('utf-8').splitlines()
524 keys = []
525 pnmap = {}
526 def readsigs(lines):
527 sigs = {}
528 for line in lines:
529 linesplit = line.split()
530 if len(linesplit) > 2:
531 sigs[linesplit[0]] = linesplit[2]
532 if not linesplit[0] in keys:
533 keys.append(linesplit[0])
534 pnmap[linesplit[1]] = linesplit[0].rsplit('.', 1)[0]
535 return sigs
536 adict = readsigs(alines)
537 bdict = readsigs(blines)
538 out = []
539
540 changecount = 0
541 addcount = 0
542 removecount = 0
543 if taskdiff:
544 with bb.tinfoil.Tinfoil() as tinfoil:
545 tinfoil.prepare(config_only=True)
546
547 changes = collections.OrderedDict()
548
549 def compare_hashfiles(pn, taskname, hash1, hash2):
550 hashes = [hash1, hash2]
551 hashfiles = bb.siggen.find_siginfo(pn, taskname, hashes, tinfoil.config_data)
552
553 if not taskname:
554 (pn, taskname) = pn.rsplit('.', 1)
555 pn = pnmap.get(pn, pn)
556 desc = '%s.%s' % (pn, taskname)
557
558 if len(hashfiles) == 0:
559 out.append("Unable to find matching sigdata for %s with hashes %s or %s" % (desc, hash1, hash2))
560 elif not hash1 in hashfiles:
561 out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash1))
562 elif not hash2 in hashfiles:
563 out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash2))
564 else:
565 out2 = bb.siggen.compare_sigfiles(hashfiles[hash1]['path'], hashfiles[hash2]['path'], recursecb, collapsed=True)
566 for line in out2:
567 m = hashlib.sha256()
568 m.update(line.encode('utf-8'))
569 entry = changes.get(m.hexdigest(), (line, []))
570 if desc not in entry[1]:
571 changes[m.hexdigest()] = (line, entry[1] + [desc])
572
573 # Define recursion callback
574 def recursecb(key, hash1, hash2):
575 compare_hashfiles(key, None, hash1, hash2)
576 return []
577
578 for key in keys:
579 siga = adict.get(key, None)
580 sigb = bdict.get(key, None)
581 if siga is not None and sigb is not None and siga != sigb:
582 changecount += 1
583 (pn, taskname) = key.rsplit('.', 1)
584 compare_hashfiles(pn, taskname, siga, sigb)
585 elif siga is None:
586 addcount += 1
587 elif sigb is None:
588 removecount += 1
589 for key, item in changes.items():
590 line, tasks = item
591 if len(tasks) == 1:
592 desc = tasks[0]
593 elif len(tasks) == 2:
594 desc = '%s and %s' % (tasks[0], tasks[1])
595 else:
596 desc = '%s and %d others' % (tasks[-1], len(tasks)-1)
597 out.append('%s: %s' % (desc, line))
598 else:
599 for key in keys:
600 siga = adict.get(key, None)
601 sigb = bdict.get(key, None)
602 if siga is not None and sigb is not None and siga != sigb:
603 out.append('%s changed from %s to %s' % (key, siga, sigb))
604 changecount += 1
605 elif siga is None:
606 out.append('%s was added' % key)
607 addcount += 1
608 elif sigb is None:
609 out.append('%s was removed' % key)
610 removecount += 1
611 out.append('Summary: %d tasks added, %d tasks removed, %d tasks modified (%.1f%%)' % (addcount, removecount, changecount, (changecount / float(len(bdict)) * 100)))
612 return '\n'.join(out)
613
614
615def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False,
616 sigs=False, sigsdiff=False, exclude_path=None):
617 repo = git.Repo(repopath)
618 assert repo.bare == False
619 commit = repo.commit(revision1)
620 diff = commit.diff(revision2)
621
622 changes = []
623
624 if sigs or sigsdiff:
625 for d in diff.iter_change_type('M'):
626 if d.a_blob.path == 'siglist.txt':
627 changes.append(compare_siglists(d.a_blob, d.b_blob, taskdiff=sigsdiff))
628 return changes
629
630 for d in diff.iter_change_type('M'):
631 path = os.path.dirname(d.a_blob.path)
632 if path.startswith('packages/'):
633 filename = os.path.basename(d.a_blob.path)
634 if filename == 'latest':
635 changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver))
636 elif filename.startswith('latest.'):
637 chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True)
638 changes.append(chg)
639 elif filename == 'sysroot':
640 alines = d.a_blob.data_stream.read().decode('utf-8').splitlines()
641 blines = d.b_blob.data_stream.read().decode('utf-8').splitlines()
642 filechanges = compare_file_lists(alines,blines, compare_ownership=False)
643 if filechanges:
644 chg = ChangeRecord(path, filename, None, None, True)
645 chg.filechanges = filechanges
646 changes.append(chg)
647
648 elif path.startswith('images/'):
649 filename = os.path.basename(d.a_blob.path)
650 if filename in img_monitor_files:
651 if filename == 'files-in-image.txt':
652 alines = d.a_blob.data_stream.read().decode('utf-8').splitlines()
653 blines = d.b_blob.data_stream.read().decode('utf-8').splitlines()
654 filechanges = compare_file_lists(alines,blines)
655 if filechanges:
656 chg = ChangeRecord(path, filename, None, None, True)
657 chg.filechanges = filechanges
658 changes.append(chg)
659 elif filename == 'installed-package-names.txt':
660 alines = d.a_blob.data_stream.read().decode('utf-8').splitlines()
661 blines = d.b_blob.data_stream.read().decode('utf-8').splitlines()
662 filechanges = compare_lists(alines,blines)
663 if filechanges:
664 chg = ChangeRecord(path, filename, None, None, True)
665 chg.filechanges = filechanges
666 changes.append(chg)
667 else:
668 chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True)
669 changes.append(chg)
670 elif filename == 'image-info.txt':
671 changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver))
672 elif '/image-files/' in path:
673 chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True)
674 changes.append(chg)
675
676 # Look for added preinst/postinst/prerm/postrm
677 # (without reporting newly added recipes)
678 addedpkgs = []
679 addedchanges = []
680 for d in diff.iter_change_type('A'):
681 path = os.path.dirname(d.b_blob.path)
682 if path.startswith('packages/'):
683 filename = os.path.basename(d.b_blob.path)
684 if filename == 'latest':
685 addedpkgs.append(path)
686 elif filename.startswith('latest.'):
687 chg = ChangeRecord(path, filename[7:], '', d.b_blob.data_stream.read().decode('utf-8'), True)
688 addedchanges.append(chg)
689 for chg in addedchanges:
690 found = False
691 for pkg in addedpkgs:
692 if chg.path.startswith(pkg):
693 found = True
694 break
695 if not found:
696 changes.append(chg)
697
698 # Look for cleared preinst/postinst/prerm/postrm
699 for d in diff.iter_change_type('D'):
700 path = os.path.dirname(d.a_blob.path)
701 if path.startswith('packages/'):
702 filename = os.path.basename(d.a_blob.path)
703 if filename != 'latest' and filename.startswith('latest.'):
704 chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read().decode('utf-8'), '', True)
705 changes.append(chg)
706
707 # filter out unwanted paths
708 if exclude_path:
709 for chg in changes:
710 if chg.filechanges:
711 fchgs = []
712 for fchg in chg.filechanges:
713 for epath in exclude_path:
714 if fchg.path.startswith(epath):
715 break
716 else:
717 fchgs.append(fchg)
718 chg.filechanges = fchgs
719
720 if report_all:
721 return changes
722 else:
723 return [chg for chg in changes if chg.monitored]
diff --git a/meta-xilinx-core/lib/oe/cachedpath.py b/meta-xilinx-core/lib/oe/cachedpath.py
new file mode 100644
index 00000000..0138b791
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/cachedpath.py
@@ -0,0 +1,237 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6# Based on standard python library functions but avoid
7# repeated stat calls. Its assumed the files will not change from under us
8# so we can cache stat calls.
9#
10
11import os
12import errno
13import stat as statmod
14
15class CachedPath(object):
16 def __init__(self):
17 self.statcache = {}
18 self.lstatcache = {}
19 self.normpathcache = {}
20 return
21
22 def updatecache(self, x):
23 x = self.normpath(x)
24 if x in self.statcache:
25 del self.statcache[x]
26 if x in self.lstatcache:
27 del self.lstatcache[x]
28
29 def normpath(self, path):
30 if path in self.normpathcache:
31 return self.normpathcache[path]
32 newpath = os.path.normpath(path)
33 self.normpathcache[path] = newpath
34 return newpath
35
36 def _callstat(self, path):
37 if path in self.statcache:
38 return self.statcache[path]
39 try:
40 st = os.stat(path)
41 self.statcache[path] = st
42 return st
43 except os.error:
44 self.statcache[path] = False
45 return False
46
47 # We might as well call lstat and then only
48 # call stat as well in the symbolic link case
49 # since this turns out to be much more optimal
50 # in real world usage of this cache
51 def callstat(self, path):
52 path = self.normpath(path)
53 self.calllstat(path)
54 return self.statcache[path]
55
56 def calllstat(self, path):
57 path = self.normpath(path)
58 if path in self.lstatcache:
59 return self.lstatcache[path]
60 #bb.error("LStatpath:" + path)
61 try:
62 lst = os.lstat(path)
63 self.lstatcache[path] = lst
64 if not statmod.S_ISLNK(lst.st_mode):
65 self.statcache[path] = lst
66 else:
67 self._callstat(path)
68 return lst
69 except (os.error, AttributeError):
70 self.lstatcache[path] = False
71 self.statcache[path] = False
72 return False
73
74 # This follows symbolic links, so both islink() and isdir() can be true
75 # for the same path ono systems that support symlinks
76 def isfile(self, path):
77 """Test whether a path is a regular file"""
78 st = self.callstat(path)
79 if not st:
80 return False
81 return statmod.S_ISREG(st.st_mode)
82
83 # Is a path a directory?
84 # This follows symbolic links, so both islink() and isdir()
85 # can be true for the same path on systems that support symlinks
86 def isdir(self, s):
87 """Return true if the pathname refers to an existing directory."""
88 st = self.callstat(s)
89 if not st:
90 return False
91 return statmod.S_ISDIR(st.st_mode)
92
93 def islink(self, path):
94 """Test whether a path is a symbolic link"""
95 st = self.calllstat(path)
96 if not st:
97 return False
98 return statmod.S_ISLNK(st.st_mode)
99
100 # Does a path exist?
101 # This is false for dangling symbolic links on systems that support them.
102 def exists(self, path):
103 """Test whether a path exists. Returns False for broken symbolic links"""
104 if self.callstat(path):
105 return True
106 return False
107
108 def lexists(self, path):
109 """Test whether a path exists. Returns True for broken symbolic links"""
110 if self.calllstat(path):
111 return True
112 return False
113
114 def stat(self, path):
115 return self.callstat(path)
116
117 def lstat(self, path):
118 return self.calllstat(path)
119
120 def walk(self, top, topdown=True, onerror=None, followlinks=False):
121 # Matches os.walk, not os.path.walk()
122
123 # We may not have read permission for top, in which case we can't
124 # get a list of the files the directory contains. os.path.walk
125 # always suppressed the exception then, rather than blow up for a
126 # minor reason when (say) a thousand readable directories are still
127 # left to visit. That logic is copied here.
128 try:
129 names = os.listdir(top)
130 except os.error as err:
131 if onerror is not None:
132 onerror(err)
133 return
134
135 dirs, nondirs = [], []
136 for name in names:
137 if self.isdir(os.path.join(top, name)):
138 dirs.append(name)
139 else:
140 nondirs.append(name)
141
142 if topdown:
143 yield top, dirs, nondirs
144 for name in dirs:
145 new_path = os.path.join(top, name)
146 if followlinks or not self.islink(new_path):
147 for x in self.walk(new_path, topdown, onerror, followlinks):
148 yield x
149 if not topdown:
150 yield top, dirs, nondirs
151
152 ## realpath() related functions
153 def __is_path_below(self, file, root):
154 return (file + os.path.sep).startswith(root)
155
156 def __realpath_rel(self, start, rel_path, root, loop_cnt, assume_dir):
157 """Calculates real path of symlink 'start' + 'rel_path' below
158 'root'; no part of 'start' below 'root' must contain symlinks. """
159 have_dir = True
160
161 for d in rel_path.split(os.path.sep):
162 if not have_dir and not assume_dir:
163 raise OSError(errno.ENOENT, "no such directory %s" % start)
164
165 if d == os.path.pardir: # '..'
166 if len(start) >= len(root):
167 # do not follow '..' before root
168 start = os.path.dirname(start)
169 else:
170 # emit warning?
171 pass
172 else:
173 (start, have_dir) = self.__realpath(os.path.join(start, d),
174 root, loop_cnt, assume_dir)
175
176 assert(self.__is_path_below(start, root))
177
178 return start
179
180 def __realpath(self, file, root, loop_cnt, assume_dir):
181 while self.islink(file) and len(file) >= len(root):
182 if loop_cnt == 0:
183 raise OSError(errno.ELOOP, file)
184
185 loop_cnt -= 1
186 target = os.path.normpath(os.readlink(file))
187
188 if not os.path.isabs(target):
189 tdir = os.path.dirname(file)
190 assert(self.__is_path_below(tdir, root))
191 else:
192 tdir = root
193
194 file = self.__realpath_rel(tdir, target, root, loop_cnt, assume_dir)
195
196 try:
197 is_dir = self.isdir(file)
198 except:
199 is_dir = False
200
201 return (file, is_dir)
202
203 def realpath(self, file, root, use_physdir = True, loop_cnt = 100, assume_dir = False):
204 """ Returns the canonical path of 'file' with assuming a
205 toplevel 'root' directory. When 'use_physdir' is set, all
206 preceding path components of 'file' will be resolved first;
207 this flag should be set unless it is guaranteed that there is
208 no symlink in the path. When 'assume_dir' is not set, missing
209 path components will raise an ENOENT error"""
210
211 root = os.path.normpath(root)
212 file = os.path.normpath(file)
213
214 if not root.endswith(os.path.sep):
215 # letting root end with '/' makes some things easier
216 root = root + os.path.sep
217
218 if not self.__is_path_below(file, root):
219 raise OSError(errno.EINVAL, "file '%s' is not below root" % file)
220
221 try:
222 if use_physdir:
223 file = self.__realpath_rel(root, file[(len(root) - 1):], root, loop_cnt, assume_dir)
224 else:
225 file = self.__realpath(file, root, loop_cnt, assume_dir)[0]
226 except OSError as e:
227 if e.errno == errno.ELOOP:
228 # make ELOOP more readable; without catching it, there will
229 # be printed a backtrace with 100s of OSError exceptions
230 # else
231 raise OSError(errno.ELOOP,
232 "too much recursions while resolving '%s'; loop in '%s'" %
233 (file, e.strerror))
234
235 raise
236
237 return file
diff --git a/meta-xilinx-core/lib/oe/classextend.py b/meta-xilinx-core/lib/oe/classextend.py
new file mode 100644
index 00000000..5161d33d
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/classextend.py
@@ -0,0 +1,159 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import collections
8
9def get_packages(d):
10 pkgs = d.getVar("PACKAGES_NONML")
11 extcls = d.getVar("EXTENDERCLASS")
12 return extcls.rename_packages_internal(pkgs)
13
14def get_depends(varprefix, d):
15 extcls = d.getVar("EXTENDERCLASS")
16 return extcls.map_depends_variable(varprefix + "_NONML")
17
18class ClassExtender(object):
19 def __init__(self, extname, d):
20 self.extname = extname
21 self.d = d
22 self.pkgs_mapping = []
23 self.d.setVar("EXTENDERCLASS", self)
24
25 def extend_name(self, name):
26 if name.startswith("kernel-") or name == "virtual/kernel":
27 return name
28 if name.startswith("rtld"):
29 return name
30 if name.endswith("-crosssdk"):
31 return name
32 if name.endswith("-" + self.extname):
33 name = name.replace("-" + self.extname, "")
34 if name.startswith("virtual/"):
35 # Assume large numbers of dashes means a triplet is present and we don't need to convert
36 if name.count("-") >= 3 and name.endswith(("-go", "-binutils", "-gcc", "-g++")):
37 return name
38 subs = name.split("/", 1)[1]
39 if not subs.startswith(self.extname):
40 return "virtual/" + self.extname + "-" + subs
41 return name
42 if name.startswith("/") or (name.startswith("${") and name.endswith("}")):
43 return name
44 if not name.startswith(self.extname):
45 return self.extname + "-" + name
46 return name
47
48 def map_variable(self, varname, setvar = True):
49 var = self.d.getVar(varname)
50 if not var:
51 return ""
52 var = var.split()
53 newvar = []
54 for v in var:
55 newvar.append(self.extend_name(v))
56 newdata = " ".join(newvar)
57 if setvar:
58 self.d.setVar(varname, newdata)
59 return newdata
60
61 def map_regexp_variable(self, varname, setvar = True):
62 var = self.d.getVar(varname)
63 if not var:
64 return ""
65 var = var.split()
66 newvar = []
67 for v in var:
68 if v.startswith("^" + self.extname):
69 newvar.append(v)
70 elif v.startswith("^"):
71 newvar.append("^" + self.extname + "-" + v[1:])
72 else:
73 newvar.append(self.extend_name(v))
74 newdata = " ".join(newvar)
75 if setvar:
76 self.d.setVar(varname, newdata)
77 return newdata
78
79 def map_depends(self, dep):
80 if dep.endswith(("-native", "-native-runtime")) or ('nativesdk-' in dep) or ('cross-canadian' in dep) or ('-crosssdk-' in dep):
81 return dep
82 else:
83 # Do not extend for that already have multilib prefix
84 var = self.d.getVar("MULTILIB_VARIANTS")
85 if var:
86 var = var.split()
87 for v in var:
88 if dep.startswith(v):
89 return dep
90 return self.extend_name(dep)
91
92 def map_depends_variable(self, varname, suffix = ""):
93 # We need to preserve EXTENDPKGV so it can be expanded correctly later
94 if suffix:
95 varname = varname + ":" + suffix
96 orig = self.d.getVar("EXTENDPKGV", False)
97 self.d.setVar("EXTENDPKGV", "EXTENDPKGV")
98 deps = self.d.getVar(varname)
99 if not deps:
100 self.d.setVar("EXTENDPKGV", orig)
101 return
102 deps = bb.utils.explode_dep_versions2(deps)
103 newdeps = collections.OrderedDict()
104 for dep in deps:
105 newdeps[self.map_depends(dep)] = deps[dep]
106
107 if not varname.endswith("_NONML"):
108 self.d.renameVar(varname, varname + "_NONML")
109 self.d.setVar(varname, "${@oe.classextend.get_depends('%s', d)}" % varname)
110 self.d.appendVarFlag(varname, "vardeps", " " + varname + "_NONML")
111 ret = bb.utils.join_deps(newdeps, False).replace("EXTENDPKGV", "${EXTENDPKGV}")
112 self.d.setVar("EXTENDPKGV", orig)
113 return ret
114
115 def map_packagevars(self):
116 for pkg in (self.d.getVar("PACKAGES").split() + [""]):
117 self.map_depends_variable("RDEPENDS", pkg)
118 self.map_depends_variable("RRECOMMENDS", pkg)
119 self.map_depends_variable("RSUGGESTS", pkg)
120 self.map_depends_variable("RPROVIDES", pkg)
121 self.map_depends_variable("RREPLACES", pkg)
122 self.map_depends_variable("RCONFLICTS", pkg)
123 self.map_depends_variable("PKG", pkg)
124
125 def rename_packages(self):
126 for pkg in (self.d.getVar("PACKAGES") or "").split():
127 if pkg.startswith(self.extname):
128 self.pkgs_mapping.append([pkg.split(self.extname + "-")[1], pkg])
129 continue
130 self.pkgs_mapping.append([pkg, self.extend_name(pkg)])
131
132 self.d.renameVar("PACKAGES", "PACKAGES_NONML")
133 self.d.setVar("PACKAGES", "${@oe.classextend.get_packages(d)}")
134
135 def rename_packages_internal(self, pkgs):
136 self.pkgs_mapping = []
137 for pkg in (self.d.expand(pkgs) or "").split():
138 if pkg.startswith(self.extname):
139 self.pkgs_mapping.append([pkg.split(self.extname + "-")[1], pkg])
140 continue
141 self.pkgs_mapping.append([pkg, self.extend_name(pkg)])
142
143 return " ".join([row[1] for row in self.pkgs_mapping])
144
145 def rename_package_variables(self, variables):
146 for pkg_mapping in self.pkgs_mapping:
147 if pkg_mapping[0].startswith("${") and pkg_mapping[0].endswith("}"):
148 continue
149 for subs in variables:
150 self.d.renameVar("%s:%s" % (subs, pkg_mapping[0]), "%s:%s" % (subs, pkg_mapping[1]))
151
152class NativesdkClassExtender(ClassExtender):
153 def map_depends(self, dep):
154 if dep.startswith(self.extname):
155 return dep
156 if dep.endswith(("-native", "-native-runtime")) or ('nativesdk-' in dep) or ('-cross-' in dep) or ('-crosssdk-' in dep):
157 return dep
158 else:
159 return self.extend_name(dep)
diff --git a/meta-xilinx-core/lib/oe/classutils.py b/meta-xilinx-core/lib/oe/classutils.py
new file mode 100644
index 00000000..ec3f6ad7
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/classutils.py
@@ -0,0 +1,49 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7class ClassRegistryMeta(type):
8 """Give each ClassRegistry their own registry"""
9 def __init__(cls, name, bases, attrs):
10 cls.registry = {}
11 type.__init__(cls, name, bases, attrs)
12
13class ClassRegistry(type, metaclass=ClassRegistryMeta):
14 """Maintain a registry of classes, indexed by name.
15
16Note that this implementation requires that the names be unique, as it uses
17a dictionary to hold the classes by name.
18
19The name in the registry can be overridden via the 'name' attribute of the
20class, and the 'priority' attribute controls priority. The prioritized()
21method returns the registered classes in priority order.
22
23Subclasses of ClassRegistry may define an 'implemented' property to exert
24control over whether the class will be added to the registry (e.g. to keep
25abstract base classes out of the registry)."""
26 priority = 0
27 def __init__(cls, name, bases, attrs):
28 super(ClassRegistry, cls).__init__(name, bases, attrs)
29 try:
30 if not cls.implemented:
31 return
32 except AttributeError:
33 pass
34
35 try:
36 cls.name
37 except AttributeError:
38 cls.name = name
39 cls.registry[cls.name] = cls
40
41 @classmethod
42 def prioritized(tcls):
43 return sorted(list(tcls.registry.values()),
44 key=lambda v: (v.priority, v.name), reverse=True)
45
46 def unregister(cls):
47 for key in cls.registry.keys():
48 if cls.registry[key] is cls:
49 del cls.registry[key]
diff --git a/meta-xilinx-core/lib/oe/copy_buildsystem.py b/meta-xilinx-core/lib/oe/copy_buildsystem.py
new file mode 100644
index 00000000..81abfbf9
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/copy_buildsystem.py
@@ -0,0 +1,293 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6# This class should provide easy access to the different aspects of the
7# buildsystem such as layers, bitbake location, etc.
8#
9# SDK_LAYERS_EXCLUDE: Layers which will be excluded from SDK layers.
10# SDK_LAYERS_EXCLUDE_PATTERN: The simiar to SDK_LAYERS_EXCLUDE, this supports
11# python regular expression, use space as separator,
12# e.g.: ".*-downloads closed-.*"
13#
14
15import stat
16import shutil
17
18def _smart_copy(src, dest):
19 import subprocess
20 # smart_copy will choose the correct function depending on whether the
21 # source is a file or a directory.
22 mode = os.stat(src).st_mode
23 if stat.S_ISDIR(mode):
24 bb.utils.mkdirhier(dest)
25 cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs --xattrs-include='*' -cf - -C %s -p . \
26 | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)
27 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
28 else:
29 shutil.copyfile(src, dest)
30 shutil.copymode(src, dest)
31
32class BuildSystem(object):
33 def __init__(self, context, d):
34 self.d = d
35 self.context = context
36 self.layerdirs = [os.path.abspath(pth) for pth in d.getVar('BBLAYERS').split()]
37 self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE') or "").split()
38 self.layers_exclude_pattern = d.getVar('SDK_LAYERS_EXCLUDE_PATTERN')
39
40 def copy_bitbake_and_layers(self, destdir, workspace_name=None):
41 import re
42 # Copy in all metadata layers + bitbake (as repositories)
43 copied_corebase = None
44 layers_copied = []
45 bb.utils.mkdirhier(destdir)
46 layers = list(self.layerdirs)
47
48 corebase = os.path.abspath(self.d.getVar('COREBASE'))
49 layers.append(corebase)
50 # The bitbake build system uses the meta-skeleton layer as a layout
51 # for common recipies, e.g: the recipetool script to create kernel recipies
52 # Add the meta-skeleton layer to be included as part of the eSDK installation
53 layers.append(os.path.join(corebase, 'meta-skeleton'))
54
55 # Exclude layers
56 for layer_exclude in self.layers_exclude:
57 if layer_exclude in layers:
58 bb.note('Excluded %s from sdk layers since it is in SDK_LAYERS_EXCLUDE' % layer_exclude)
59 layers.remove(layer_exclude)
60
61 if self.layers_exclude_pattern:
62 layers_cp = layers[:]
63 for pattern in self.layers_exclude_pattern.split():
64 for layer in layers_cp:
65 if re.match(pattern, layer):
66 bb.note('Excluded %s from sdk layers since matched SDK_LAYERS_EXCLUDE_PATTERN' % layer)
67 layers.remove(layer)
68
69 workspace_newname = workspace_name
70 if workspace_newname:
71 layernames = [os.path.basename(layer) for layer in layers]
72 extranum = 0
73 while workspace_newname in layernames:
74 extranum += 1
75 workspace_newname = '%s-%d' % (workspace_name, extranum)
76
77 corebase_files = self.d.getVar('COREBASE_FILES').split()
78 corebase_files = [corebase + '/' +x for x in corebase_files]
79 # Make sure bitbake goes in
80 bitbake_dir = bb.__file__.rsplit('/', 3)[0]
81 corebase_files.append(bitbake_dir)
82
83 for layer in layers:
84 layerconf = os.path.join(layer, 'conf', 'layer.conf')
85 layernewname = os.path.basename(layer)
86 workspace = False
87 if os.path.exists(layerconf):
88 with open(layerconf, 'r') as f:
89 if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"):
90 if workspace_newname:
91 layernewname = workspace_newname
92 workspace = True
93 else:
94 bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context))
95 continue
96
97 # If the layer was already under corebase, leave it there
98 # since layers such as meta have issues when moved.
99 layerdestpath = destdir
100 if corebase == os.path.dirname(layer):
101 layerdestpath += '/' + os.path.basename(corebase)
102 # If the layer is located somewhere under the same parent directory
103 # as corebase we keep the layer structure.
104 elif os.path.commonpath([layer, corebase]) == os.path.dirname(corebase):
105 layer_relative = os.path.relpath(layer, os.path.dirname(corebase))
106 if os.path.dirname(layer_relative) != layernewname:
107 layerdestpath += '/' + os.path.dirname(layer_relative)
108
109 layerdestpath += '/' + layernewname
110
111 layer_relative = os.path.relpath(layerdestpath,
112 destdir)
113 # Treat corebase as special since it typically will contain
114 # build directories or other custom items.
115 if corebase == layer:
116 copied_corebase = layer_relative
117 bb.utils.mkdirhier(layerdestpath)
118 for f in corebase_files:
119 f_basename = os.path.basename(f)
120 destname = os.path.join(layerdestpath, f_basename)
121 _smart_copy(f, destname)
122 else:
123 layers_copied.append(layer_relative)
124
125 if os.path.exists(os.path.join(layerdestpath, 'conf/layer.conf')):
126 bb.note("Skipping layer %s, already handled" % layer)
127 else:
128 _smart_copy(layer, layerdestpath)
129
130 if workspace:
131 # Make some adjustments original workspace layer
132 # Drop sources (recipe tasks will be locked, so we don't need them)
133 srcdir = os.path.join(layerdestpath, 'sources')
134 if os.path.isdir(srcdir):
135 shutil.rmtree(srcdir)
136 # Drop all bbappends except the one for the image the SDK is being built for
137 # (because of externalsrc, the workspace bbappends will interfere with the
138 # locked signatures if present, and we don't need them anyway)
139 image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE')))[0] + '.bbappend'
140 appenddir = os.path.join(layerdestpath, 'appends')
141 if os.path.isdir(appenddir):
142 for fn in os.listdir(appenddir):
143 if fn == image_bbappend:
144 continue
145 else:
146 os.remove(os.path.join(appenddir, fn))
147 # Drop README
148 readme = os.path.join(layerdestpath, 'README')
149 if os.path.exists(readme):
150 os.remove(readme)
151 # Filter out comments in layer.conf and change layer name
152 layerconf = os.path.join(layerdestpath, 'conf', 'layer.conf')
153 with open(layerconf, 'r') as f:
154 origlines = f.readlines()
155 with open(layerconf, 'w') as f:
156 for line in origlines:
157 if line.startswith('#'):
158 continue
159 line = line.replace('workspacelayer', workspace_newname)
160 f.write(line)
161
162 # meta-skeleton layer is added as part of the build system
163 # but not as a layer included in the build, therefore it is
164 # not reported to the function caller.
165 for layer in layers_copied:
166 if layer.endswith('/meta-skeleton'):
167 layers_copied.remove(layer)
168 break
169
170 return copied_corebase, layers_copied
171
172def generate_locked_sigs(sigfile, d):
173 bb.utils.mkdirhier(os.path.dirname(sigfile))
174 depd = d.getVar('BB_TASKDEPDATA', False)
175 tasks = ['%s:%s' % (v[2], v[1]) for v in depd.values()]
176 bb.parse.siggen.dump_lockedsigs(sigfile, tasks)
177
178def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, onlynative, pruned_output):
179 with open(lockedsigs, 'r') as infile:
180 bb.utils.mkdirhier(os.path.dirname(pruned_output))
181 with open(pruned_output, 'w') as f:
182 invalue = False
183 for line in infile:
184 if invalue:
185 if line.endswith('\\\n'):
186 splitval = line.strip().split(':')
187 if not splitval[1] in excluded_tasks and not splitval[0] in excluded_targets:
188 if onlynative:
189 if 'nativesdk' in splitval[0]:
190 f.write(line)
191 else:
192 f.write(line)
193 else:
194 f.write(line)
195 invalue = False
196 elif line.startswith('SIGGEN_LOCKEDSIGS'):
197 invalue = True
198 f.write(line)
199
200def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output=None):
201 merged = {}
202 arch_order = []
203 with open(lockedsigs_main, 'r') as f:
204 invalue = None
205 for line in f:
206 if invalue:
207 if line.endswith('\\\n'):
208 merged[invalue].append(line)
209 else:
210 invalue = None
211 elif line.startswith('SIGGEN_LOCKEDSIGS_t-'):
212 invalue = line[18:].split('=', 1)[0].rstrip()
213 merged[invalue] = []
214 arch_order.append(invalue)
215
216 with open(lockedsigs_extra, 'r') as f:
217 invalue = None
218 tocopy = {}
219 for line in f:
220 if invalue:
221 if line.endswith('\\\n'):
222 if not line in merged[invalue]:
223 target, task = line.strip().split(':')[:2]
224 if not copy_tasks or task in copy_tasks:
225 tocopy[invalue].append(line)
226 merged[invalue].append(line)
227 else:
228 invalue = None
229 elif line.startswith('SIGGEN_LOCKEDSIGS_t-'):
230 invalue = line[18:].split('=', 1)[0].rstrip()
231 if not invalue in merged:
232 merged[invalue] = []
233 arch_order.append(invalue)
234 tocopy[invalue] = []
235
236 def write_sigs_file(fn, types, sigs):
237 fulltypes = []
238 bb.utils.mkdirhier(os.path.dirname(fn))
239 with open(fn, 'w') as f:
240 for typename in types:
241 lines = sigs[typename]
242 if lines:
243 f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % typename)
244 for line in lines:
245 f.write(line)
246 f.write(' "\n')
247 fulltypes.append(typename)
248 f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes))
249
250 if copy_output:
251 write_sigs_file(copy_output, list(tocopy.keys()), tocopy)
252 if merged_output:
253 write_sigs_file(merged_output, arch_order, merged)
254
255def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None):
256 import shutil
257 bb.note('Generating sstate-cache...')
258
259 nativelsbstring = d.getVar('NATIVELSBSTRING')
260 bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
261 if fixedlsbstring and nativelsbstring != fixedlsbstring:
262 nativedir = output_sstate_cache + '/' + nativelsbstring
263 if os.path.isdir(nativedir):
264 destdir = os.path.join(output_sstate_cache, fixedlsbstring)
265 for root, _, files in os.walk(nativedir):
266 for fn in files:
267 src = os.path.join(root, fn)
268 dest = os.path.join(destdir, os.path.relpath(src, nativedir))
269 if os.path.exists(dest):
270 # Already exists, and it'll be the same file, so just delete it
271 os.unlink(src)
272 else:
273 bb.utils.mkdirhier(os.path.dirname(dest))
274 shutil.move(src, dest)
275
276def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None):
277 import subprocess
278
279 bb.note('Generating sstate task list...')
280
281 if not cwd:
282 cwd = os.getcwd()
283 if logfile:
284 logparam = '-l %s' % logfile
285 else:
286 logparam = ''
287 cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
288 env = dict(d.getVar('BB_ORIGENV', False))
289 env.pop('BUILDDIR', '')
290 env.pop('BBPATH', '')
291 pathitems = env['PATH'].split(':')
292 env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')])
293 bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash')
diff --git a/meta-xilinx-core/lib/oe/cve_check.py b/meta-xilinx-core/lib/oe/cve_check.py
new file mode 100644
index 00000000..ed5c714c
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/cve_check.py
@@ -0,0 +1,245 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: MIT
5#
6
7import collections
8import re
9import itertools
10import functools
11
12_Version = collections.namedtuple(
13 "_Version", ["release", "patch_l", "pre_l", "pre_v"]
14)
15
16@functools.total_ordering
17class Version():
18
19 def __init__(self, version, suffix=None):
20
21 suffixes = ["alphabetical", "patch"]
22
23 if str(suffix) == "alphabetical":
24 version_pattern = r"""r?v?(?:(?P<release>[0-9]+(?:[-\.][0-9]+)*)(?P<patch>[-_\.]?(?P<patch_l>[a-z]))?(?P<pre>[-_\.]?(?P<pre_l>(rc|alpha|beta|pre|preview|dev))[-_\.]?(?P<pre_v>[0-9]+)?)?)(.*)?"""
25 elif str(suffix) == "patch":
26 version_pattern = r"""r?v?(?:(?P<release>[0-9]+(?:[-\.][0-9]+)*)(?P<patch>[-_\.]?(p|patch)(?P<patch_l>[0-9]+))?(?P<pre>[-_\.]?(?P<pre_l>(rc|alpha|beta|pre|preview|dev))[-_\.]?(?P<pre_v>[0-9]+)?)?)(.*)?"""
27 else:
28 version_pattern = r"""r?v?(?:(?P<release>[0-9]+(?:[-\.][0-9]+)*)(?P<pre>[-_\.]?(?P<pre_l>(rc|alpha|beta|pre|preview|dev))[-_\.]?(?P<pre_v>[0-9]+)?)?)(.*)?"""
29 regex = re.compile(r"^\s*" + version_pattern + r"\s*$", re.VERBOSE | re.IGNORECASE)
30
31 match = regex.search(version)
32 if not match:
33 raise Exception("Invalid version: '{0}'".format(version))
34
35 self._version = _Version(
36 release=tuple(int(i) for i in match.group("release").replace("-",".").split(".")),
37 patch_l=match.group("patch_l") if str(suffix) in suffixes and match.group("patch_l") else "",
38 pre_l=match.group("pre_l"),
39 pre_v=match.group("pre_v")
40 )
41
42 self._key = _cmpkey(
43 self._version.release,
44 self._version.patch_l,
45 self._version.pre_l,
46 self._version.pre_v
47 )
48
49 def __eq__(self, other):
50 if not isinstance(other, Version):
51 return NotImplemented
52 return self._key == other._key
53
54 def __gt__(self, other):
55 if not isinstance(other, Version):
56 return NotImplemented
57 return self._key > other._key
58
59def _cmpkey(release, patch_l, pre_l, pre_v):
60 # remove leading 0
61 _release = tuple(
62 reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release))))
63 )
64
65 _patch = patch_l.upper()
66
67 if pre_l is None and pre_v is None:
68 _pre = float('inf')
69 else:
70 _pre = float(pre_v) if pre_v else float('-inf')
71 return _release, _patch, _pre
72
73
74def get_patched_cves(d):
75 """
76 Get patches that solve CVEs using the "CVE: " tag.
77 """
78
79 import re
80 import oe.patch
81
82 cve_match = re.compile(r"CVE:( CVE-\d{4}-\d+)+")
83
84 # Matches the last "CVE-YYYY-ID" in the file name, also if written
85 # in lowercase. Possible to have multiple CVE IDs in a single
86 # file name, but only the last one will be detected from the file name.
87 # However, patch files contents addressing multiple CVE IDs are supported
88 # (cve_match regular expression)
89 cve_file_name_match = re.compile(r".*(CVE-\d{4}-\d+)", re.IGNORECASE)
90
91 patched_cves = set()
92 patches = oe.patch.src_patches(d)
93 bb.debug(2, "Scanning %d patches for CVEs" % len(patches))
94 for url in patches:
95 patch_file = bb.fetch.decodeurl(url)[2]
96
97 # Check patch file name for CVE ID
98 fname_match = cve_file_name_match.search(patch_file)
99 if fname_match:
100 cve = fname_match.group(1).upper()
101 patched_cves.add(cve)
102 bb.debug(2, "Found %s from patch file name %s" % (cve, patch_file))
103
104 # Remote patches won't be present and compressed patches won't be
105 # unpacked, so say we're not scanning them
106 if not os.path.isfile(patch_file):
107 bb.note("%s is remote or compressed, not scanning content" % patch_file)
108 continue
109
110 with open(patch_file, "r", encoding="utf-8") as f:
111 try:
112 patch_text = f.read()
113 except UnicodeDecodeError:
114 bb.debug(1, "Failed to read patch %s using UTF-8 encoding"
115 " trying with iso8859-1" % patch_file)
116 f.close()
117 with open(patch_file, "r", encoding="iso8859-1") as f:
118 patch_text = f.read()
119
120 # Search for one or more "CVE: " lines
121 text_match = False
122 for match in cve_match.finditer(patch_text):
123 # Get only the CVEs without the "CVE: " tag
124 cves = patch_text[match.start()+5:match.end()]
125 for cve in cves.split():
126 bb.debug(2, "Patch %s solves %s" % (patch_file, cve))
127 patched_cves.add(cve)
128 text_match = True
129
130 if not fname_match and not text_match:
131 bb.debug(2, "Patch %s doesn't solve CVEs" % patch_file)
132
133 # Search for additional patched CVEs
134 for cve in (d.getVarFlags("CVE_STATUS") or {}):
135 decoded_status, _, _ = decode_cve_status(d, cve)
136 if decoded_status == "Patched":
137 bb.debug(2, "CVE %s is additionally patched" % cve)
138 patched_cves.add(cve)
139
140 return patched_cves
141
142
143def get_cpe_ids(cve_product, version):
144 """
145 Get list of CPE identifiers for the given product and version
146 """
147
148 version = version.split("+git")[0]
149
150 cpe_ids = []
151 for product in cve_product.split():
152 # CVE_PRODUCT in recipes may include vendor information for CPE identifiers. If not,
153 # use wildcard for vendor.
154 if ":" in product:
155 vendor, product = product.split(":", 1)
156 else:
157 vendor = "*"
158
159 cpe_id = 'cpe:2.3:*:{}:{}:{}:*:*:*:*:*:*:*'.format(vendor, product, version)
160 cpe_ids.append(cpe_id)
161
162 return cpe_ids
163
164def cve_check_merge_jsons(output, data):
165 """
166 Merge the data in the "package" property to the main data file
167 output
168 """
169 if output["version"] != data["version"]:
170 bb.error("Version mismatch when merging JSON outputs")
171 return
172
173 for product in output["package"]:
174 if product["name"] == data["package"][0]["name"]:
175 bb.error("Error adding the same package %s twice" % product["name"])
176 return
177
178 output["package"].append(data["package"][0])
179
180def update_symlinks(target_path, link_path):
181 """
182 Update a symbolic link link_path to point to target_path.
183 Remove the link and recreate it if exist and is different.
184 """
185 if link_path != target_path and os.path.exists(target_path):
186 if os.path.exists(os.path.realpath(link_path)):
187 os.remove(link_path)
188 os.symlink(os.path.basename(target_path), link_path)
189
190
191def convert_cve_version(version):
192 """
193 This function converts from CVE format to Yocto version format.
194 eg 8.3_p1 -> 8.3p1, 6.2_rc1 -> 6.2-rc1
195
196 Unless it is redefined using CVE_VERSION in the recipe,
197 cve_check uses the version in the name of the recipe (${PV})
198 to check vulnerabilities against a CVE in the database downloaded from NVD.
199
200 When the version has an update, i.e.
201 "p1" in OpenSSH 8.3p1,
202 "-rc1" in linux kernel 6.2-rc1,
203 the database stores the version as version_update (8.3_p1, 6.2_rc1).
204 Therefore, we must transform this version before comparing to the
205 recipe version.
206
207 In this case, the parameter of the function is 8.3_p1.
208 If the version uses the Release Candidate format, "rc",
209 this function replaces the '_' by '-'.
210 If the version uses the Update format, "p",
211 this function removes the '_' completely.
212 """
213 import re
214
215 matches = re.match('^([0-9.]+)_((p|rc)[0-9]+)$', version)
216
217 if not matches:
218 return version
219
220 version = matches.group(1)
221 update = matches.group(2)
222
223 if matches.group(3) == "rc":
224 return version + '-' + update
225
226 return version + update
227
228def decode_cve_status(d, cve):
229 """
230 Convert CVE_STATUS into status, detail and description.
231 """
232 status = d.getVarFlag("CVE_STATUS", cve)
233 if not status:
234 return ("", "", "")
235
236 status_split = status.split(':', 1)
237 detail = status_split[0]
238 description = status_split[1].strip() if (len(status_split) > 1) else ""
239
240 status_mapping = d.getVarFlag("CVE_CHECK_STATUSMAP", detail)
241 if status_mapping is None:
242 bb.warn('Invalid detail "%s" for CVE_STATUS[%s] = "%s", fallback to Unpatched' % (detail, cve, status))
243 status_mapping = "Unpatched"
244
245 return (status_mapping, detail, description)
diff --git a/meta-xilinx-core/lib/oe/data.py b/meta-xilinx-core/lib/oe/data.py
new file mode 100644
index 00000000..37121cfa
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/data.py
@@ -0,0 +1,53 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import json
8import oe.maketype
9
10def typed_value(key, d):
11 """Construct a value for the specified metadata variable, using its flags
12 to determine the type and parameters for construction."""
13 var_type = d.getVarFlag(key, 'type')
14 flags = d.getVarFlags(key)
15 if flags is not None:
16 flags = dict((flag, d.expand(value))
17 for flag, value in list(flags.items()))
18 else:
19 flags = {}
20
21 try:
22 return oe.maketype.create(d.getVar(key) or '', var_type, **flags)
23 except (TypeError, ValueError) as exc:
24 bb.msg.fatal("Data", "%s: %s" % (key, str(exc)))
25
26def export2json(d, json_file, expand=True, searchString="",replaceString=""):
27 data2export = {}
28 keys2export = []
29
30 for key in d.keys():
31 if key.startswith("_"):
32 continue
33 elif key.startswith("BB"):
34 continue
35 elif key.startswith("B_pn"):
36 continue
37 elif key.startswith("do_"):
38 continue
39 elif d.getVarFlag(key, "func"):
40 continue
41
42 keys2export.append(key)
43
44 for key in keys2export:
45 try:
46 data2export[key] = d.getVar(key, expand).replace(searchString,replaceString)
47 except bb.data_smart.ExpansionError:
48 data2export[key] = ''
49 except AttributeError:
50 pass
51
52 with open(json_file, "w") as f:
53 json.dump(data2export, f, skipkeys=True, indent=4, sort_keys=True)
diff --git a/meta-xilinx-core/lib/oe/distro_check.py b/meta-xilinx-core/lib/oe/distro_check.py
new file mode 100644
index 00000000..3494520f
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/distro_check.py
@@ -0,0 +1,314 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7def create_socket(url, d):
8 import urllib
9 from bb.utils import export_proxies
10
11 export_proxies(d)
12 return urllib.request.urlopen(url)
13
14def get_links_from_url(url, d):
15 "Return all the href links found on the web location"
16
17 from bs4 import BeautifulSoup, SoupStrainer
18
19 soup = BeautifulSoup(create_socket(url,d), "html.parser", parse_only=SoupStrainer("a"))
20 hyperlinks = []
21 for line in soup.find_all('a', href=True):
22 hyperlinks.append(line['href'].strip('/'))
23 return hyperlinks
24
25def find_latest_numeric_release(url, d):
26 "Find the latest listed numeric release on the given url"
27 max=0
28 maxstr=""
29 for link in get_links_from_url(url, d):
30 try:
31 # TODO use bb.utils.vercmp_string_op()
32 release = float(link)
33 except:
34 release = 0
35 if release > max:
36 max = release
37 maxstr = link
38 return maxstr
39
40def is_src_rpm(name):
41 "Check if the link is pointing to a src.rpm file"
42 return name.endswith(".src.rpm")
43
44def package_name_from_srpm(srpm):
45 "Strip out the package name from the src.rpm filename"
46
47 # ca-certificates-2016.2.7-1.0.fc24.src.rpm
48 # ^name ^ver ^release^removed
49 (name, version, release) = srpm.replace(".src.rpm", "").rsplit("-", 2)
50 return name
51
52def get_source_package_list_from_url(url, section, d):
53 "Return a sectioned list of package names from a URL list"
54
55 bb.note("Reading %s: %s" % (url, section))
56 links = get_links_from_url(url, d)
57 srpms = filter(is_src_rpm, links)
58 names_list = map(package_name_from_srpm, srpms)
59
60 new_pkgs = set()
61 for pkgs in names_list:
62 new_pkgs.add(pkgs + ":" + section)
63 return new_pkgs
64
65def get_source_package_list_from_url_by_letter(url, section, d):
66 import string
67 from urllib.error import HTTPError
68 packages = set()
69 for letter in (string.ascii_lowercase + string.digits):
70 # Not all subfolders may exist, so silently handle 404
71 try:
72 packages |= get_source_package_list_from_url(url + "/" + letter, section, d)
73 except HTTPError as e:
74 if e.code != 404: raise
75 return packages
76
77def get_latest_released_fedora_source_package_list(d):
78 "Returns list of all the name os packages in the latest fedora distro"
79 latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d)
80 package_names = get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Everything/source/tree/Packages/" % latest, "main", d)
81 package_names |= get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d)
82 return latest, package_names
83
84def get_latest_released_opensuse_source_package_list(d):
85 "Returns list of all the name os packages in the latest opensuse distro"
86 latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/leap", d)
87
88 package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/leap/%s/repo/oss/suse/src/" % latest, "main", d)
89 package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/leap/%s/oss/src/" % latest, "updates", d)
90 return latest, package_names
91
92def get_latest_released_clear_source_package_list(d):
93 latest = find_latest_numeric_release("https://download.clearlinux.org/releases/", d)
94 package_names = get_source_package_list_from_url("https://download.clearlinux.org/releases/%s/clear/source/SRPMS/" % latest, "main", d)
95 return latest, package_names
96
97def find_latest_debian_release(url, d):
98 "Find the latest listed debian release on the given url"
99
100 releases = [link.replace("Debian", "")
101 for link in get_links_from_url(url, d)
102 if link.startswith("Debian")]
103 releases.sort()
104 try:
105 return releases[-1]
106 except:
107 return "_NotFound_"
108
109def get_debian_style_source_package_list(url, section, d):
110 "Return the list of package-names stored in the debian style Sources.gz file"
111 import gzip
112
113 package_names = set()
114 for line in gzip.open(create_socket(url, d), mode="rt"):
115 if line.startswith("Package:"):
116 pkg = line.split(":", 1)[1].strip()
117 package_names.add(pkg + ":" + section)
118 return package_names
119
120def get_latest_released_debian_source_package_list(d):
121 "Returns list of all the name of packages in the latest debian distro"
122 latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d)
123 url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz"
124 package_names = get_debian_style_source_package_list(url, "main", d)
125 url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz"
126 package_names |= get_debian_style_source_package_list(url, "updates", d)
127 return latest, package_names
128
129def find_latest_ubuntu_release(url, d):
130 """
131 Find the latest listed Ubuntu release on the given ubuntu/dists/ URL.
132
133 To avoid matching development releases look for distributions that have
134 updates, so the resulting distro could be any supported release.
135 """
136 url += "?C=M;O=D" # Descending Sort by Last Modified
137 for link in get_links_from_url(url, d):
138 if "-updates" in link:
139 distro = link.replace("-updates", "")
140 return distro
141 return "_NotFound_"
142
143def get_latest_released_ubuntu_source_package_list(d):
144 "Returns list of all the name os packages in the latest ubuntu distro"
145 latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d)
146 url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest
147 package_names = get_debian_style_source_package_list(url, "main", d)
148 url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest
149 package_names |= get_debian_style_source_package_list(url, "updates", d)
150 return latest, package_names
151
152def create_distro_packages_list(distro_check_dir, d):
153 import shutil
154
155 pkglst_dir = os.path.join(distro_check_dir, "package_lists")
156 bb.utils.remove(pkglst_dir, True)
157 bb.utils.mkdirhier(pkglst_dir)
158
159 per_distro_functions = (
160 ("Debian", get_latest_released_debian_source_package_list),
161 ("Ubuntu", get_latest_released_ubuntu_source_package_list),
162 ("Fedora", get_latest_released_fedora_source_package_list),
163 ("openSUSE", get_latest_released_opensuse_source_package_list),
164 ("Clear", get_latest_released_clear_source_package_list),
165 )
166
167 for name, fetcher_func in per_distro_functions:
168 try:
169 release, package_list = fetcher_func(d)
170 except Exception as e:
171 bb.warn("Cannot fetch packages for %s: %s" % (name, e))
172 bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list)))
173 if len(package_list) == 0:
174 bb.error("Didn't fetch any packages for %s %s" % (name, release))
175
176 package_list_file = os.path.join(pkglst_dir, name + "-" + release)
177 with open(package_list_file, 'w') as f:
178 for pkg in sorted(package_list):
179 f.write(pkg + "\n")
180
181def update_distro_data(distro_check_dir, datetime, d):
182 """
183 If distro packages list data is old then rebuild it.
184 The operations has to be protected by a lock so that
185 only one thread performes it at a time.
186 """
187 if not os.path.isdir (distro_check_dir):
188 try:
189 bb.note ("Making new directory: %s" % distro_check_dir)
190 os.makedirs (distro_check_dir)
191 except OSError:
192 raise Exception('Unable to create directory %s' % (distro_check_dir))
193
194
195 datetime_file = os.path.join(distro_check_dir, "build_datetime")
196 saved_datetime = "_invalid_"
197 import fcntl
198 try:
199 if not os.path.exists(datetime_file):
200 open(datetime_file, 'w+').close() # touch the file so that the next open won't fail
201
202 f = open(datetime_file, "r+")
203 fcntl.lockf(f, fcntl.LOCK_EX)
204 saved_datetime = f.read()
205 if saved_datetime[0:8] != datetime[0:8]:
206 bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime))
207 bb.note("Regenerating distro package lists")
208 create_distro_packages_list(distro_check_dir, d)
209 f.seek(0)
210 f.write(datetime)
211
212 except OSError as e:
213 raise Exception('Unable to open timestamp: %s' % e)
214 finally:
215 fcntl.lockf(f, fcntl.LOCK_UN)
216 f.close()
217
218def compare_in_distro_packages_list(distro_check_dir, d):
219 if not os.path.isdir(distro_check_dir):
220 raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed")
221
222 localdata = bb.data.createCopy(d)
223 pkglst_dir = os.path.join(distro_check_dir, "package_lists")
224 matching_distros = []
225 pn = recipe_name = d.getVar('PN')
226 bb.note("Checking: %s" % pn)
227
228 if pn.find("-native") != -1:
229 pnstripped = pn.split("-native")
230 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES'))
231 recipe_name = pnstripped[0]
232
233 if pn.startswith("nativesdk-"):
234 pnstripped = pn.split("nativesdk-")
235 localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES'))
236 recipe_name = pnstripped[1]
237
238 if pn.find("-cross") != -1:
239 pnstripped = pn.split("-cross")
240 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES'))
241 recipe_name = pnstripped[0]
242
243 if pn.find("-initial") != -1:
244 pnstripped = pn.split("-initial")
245 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES'))
246 recipe_name = pnstripped[0]
247
248 bb.note("Recipe: %s" % recipe_name)
249
250 distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'})
251 tmp = localdata.getVar('DISTRO_PN_ALIAS') or ""
252 for str in tmp.split():
253 if str and str.find("=") == -1 and distro_exceptions[str]:
254 matching_distros.append(str)
255
256 distro_pn_aliases = {}
257 for str in tmp.split():
258 if "=" in str:
259 (dist, pn_alias) = str.split('=')
260 distro_pn_aliases[dist.strip().lower()] = pn_alias.strip()
261
262 for file in os.listdir(pkglst_dir):
263 (distro, distro_release) = file.split("-")
264 f = open(os.path.join(pkglst_dir, file), "r")
265 for line in f:
266 (pkg, section) = line.split(":")
267 if distro.lower() in distro_pn_aliases:
268 pn = distro_pn_aliases[distro.lower()]
269 else:
270 pn = recipe_name
271 if pn == pkg:
272 matching_distros.append(distro + "-" + section[:-1]) # strip the \n at the end
273 f.close()
274 break
275 f.close()
276
277 for item in tmp.split():
278 matching_distros.append(item)
279 bb.note("Matching: %s" % matching_distros)
280 return matching_distros
281
282def create_log_file(d, logname):
283 logpath = d.getVar('LOG_DIR')
284 bb.utils.mkdirhier(logpath)
285 logfn, logsuffix = os.path.splitext(logname)
286 logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME'), logsuffix))
287 if not os.path.exists(logfile):
288 slogfile = os.path.join(logpath, logname)
289 if os.path.exists(slogfile):
290 os.remove(slogfile)
291 open(logfile, 'w+').close()
292 os.symlink(logfile, slogfile)
293 d.setVar('LOG_FILE', logfile)
294 return logfile
295
296
297def save_distro_check_result(result, datetime, result_file, d):
298 pn = d.getVar('PN')
299 logdir = d.getVar('LOG_DIR')
300 if not logdir:
301 bb.error("LOG_DIR variable is not defined, can't write the distro_check results")
302 return
303 bb.utils.mkdirhier(logdir)
304
305 line = pn
306 for i in result:
307 line = line + "," + i
308 f = open(result_file, "a")
309 import fcntl
310 fcntl.lockf(f, fcntl.LOCK_EX)
311 f.seek(0, os.SEEK_END) # seek to the end of file
312 f.write(line + "\n")
313 fcntl.lockf(f, fcntl.LOCK_UN)
314 f.close()
diff --git a/meta-xilinx-core/lib/oe/elf.py b/meta-xilinx-core/lib/oe/elf.py
new file mode 100644
index 00000000..eab2349a
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/elf.py
@@ -0,0 +1,145 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7def machine_dict(d):
8# TARGET_OS TARGET_ARCH MACHINE, OSABI, ABIVERSION, Little Endian, 32bit?
9 machdata = {
10 "darwin9" : {
11 "arm" : (40, 0, 0, True, 32),
12 },
13 "eabi" : {
14 "arm" : (40, 0, 0, True, 32),
15 },
16 "elf" : {
17 "aarch64" : (183, 0, 0, True, 64),
18 "aarch64_be" :(183, 0, 0, False, 64),
19 "i586" : (3, 0, 0, True, 32),
20 "i686" : (3, 0, 0, True, 32),
21 "x86_64": (62, 0, 0, True, 64),
22 "epiphany": (4643, 0, 0, True, 32),
23 "lm32": (138, 0, 0, False, 32),
24 "loongarch64":(258, 0, 0, True, 64),
25 "mips": ( 8, 0, 0, False, 32),
26 "mipsel": ( 8, 0, 0, True, 32),
27 "microblaze": (189, 0, 0, False, 32),
28 "microblazeel":(189, 0, 0, True, 32),
29 "powerpc": (20, 0, 0, False, 32),
30 "riscv32": (243, 0, 0, True, 32),
31 "riscv64": (243, 0, 0, True, 64),
32 },
33 "linux" : {
34 "aarch64" : (183, 0, 0, True, 64),
35 "aarch64_be" :(183, 0, 0, False, 64),
36 "arm" : (40, 97, 0, True, 32),
37 "armeb": (40, 97, 0, False, 32),
38 "powerpc": (20, 0, 0, False, 32),
39 "powerpc64": (21, 0, 0, False, 64),
40 "powerpc64le": (21, 0, 0, True, 64),
41 "i386": ( 3, 0, 0, True, 32),
42 "i486": ( 3, 0, 0, True, 32),
43 "i586": ( 3, 0, 0, True, 32),
44 "i686": ( 3, 0, 0, True, 32),
45 "x86_64": (62, 0, 0, True, 64),
46 "ia64": (50, 0, 0, True, 64),
47 "alpha": (36902, 0, 0, True, 64),
48 "hppa": (15, 3, 0, False, 32),
49 "loongarch64":(258, 0, 0, True, 64),
50 "m68k": ( 4, 0, 0, False, 32),
51 "mips": ( 8, 0, 0, False, 32),
52 "mipsel": ( 8, 0, 0, True, 32),
53 "mips64": ( 8, 0, 0, False, 64),
54 "mips64el": ( 8, 0, 0, True, 64),
55 "mipsisa32r6": ( 8, 0, 0, False, 32),
56 "mipsisa32r6el": ( 8, 0, 0, True, 32),
57 "mipsisa64r6": ( 8, 0, 0, False, 64),
58 "mipsisa64r6el": ( 8, 0, 0, True, 64),
59 "nios2": (113, 0, 0, True, 32),
60 "riscv32": (243, 0, 0, True, 32),
61 "riscv64": (243, 0, 0, True, 64),
62 "s390": (22, 0, 0, False, 32),
63 "sh4": (42, 0, 0, True, 32),
64 "sparc": ( 2, 0, 0, False, 32),
65 "microblaze": (189, 0, 0, False, 32),
66 "microblazeel":(189, 0, 0, True, 32),
67 },
68 "linux-android" : {
69 "aarch64" : (183, 0, 0, True, 64),
70 "i686": ( 3, 0, 0, True, 32),
71 "x86_64": (62, 0, 0, True, 64),
72 },
73 "linux-androideabi" : {
74 "arm" : (40, 97, 0, True, 32),
75 },
76 "linux-musl" : {
77 "aarch64" : (183, 0, 0, True, 64),
78 "aarch64_be" :(183, 0, 0, False, 64),
79 "arm" : ( 40, 97, 0, True, 32),
80 "armeb": ( 40, 97, 0, False, 32),
81 "powerpc": ( 20, 0, 0, False, 32),
82 "powerpc64": ( 21, 0, 0, False, 64),
83 "powerpc64le": (21, 0, 0, True, 64),
84 "i386": ( 3, 0, 0, True, 32),
85 "i486": ( 3, 0, 0, True, 32),
86 "i586": ( 3, 0, 0, True, 32),
87 "i686": ( 3, 0, 0, True, 32),
88 "x86_64": ( 62, 0, 0, True, 64),
89 "mips": ( 8, 0, 0, False, 32),
90 "mipsel": ( 8, 0, 0, True, 32),
91 "mips64": ( 8, 0, 0, False, 64),
92 "mips64el": ( 8, 0, 0, True, 64),
93 "microblaze": (189, 0, 0, False, 32),
94 "microblazeel":(189, 0, 0, True, 32),
95 "riscv32": (243, 0, 0, True, 32),
96 "riscv64": (243, 0, 0, True, 64),
97 "sh4": ( 42, 0, 0, True, 32),
98 },
99 "uclinux-uclibc" : {
100 "bfin": ( 106, 0, 0, True, 32),
101 },
102 "linux-gnueabi" : {
103 "arm" : (40, 0, 0, True, 32),
104 "armeb" : (40, 0, 0, False, 32),
105 },
106 "linux-musleabi" : {
107 "arm" : (40, 0, 0, True, 32),
108 "armeb" : (40, 0, 0, False, 32),
109 },
110 "linux-gnuspe" : {
111 "powerpc": (20, 0, 0, False, 32),
112 },
113 "linux-muslspe" : {
114 "powerpc": (20, 0, 0, False, 32),
115 },
116 "linux-gnu" : {
117 "powerpc": (20, 0, 0, False, 32),
118 "sh4": (42, 0, 0, True, 32),
119 },
120 "linux-gnu_ilp32" : {
121 "aarch64" : (183, 0, 0, True, 32),
122 },
123 "linux-gnux32" : {
124 "x86_64": (62, 0, 0, True, 32),
125 },
126 "linux-muslx32" : {
127 "x86_64": (62, 0, 0, True, 32),
128 },
129 "linux-gnun32" : {
130 "mips64": ( 8, 0, 0, False, 32),
131 "mips64el": ( 8, 0, 0, True, 32),
132 "mipsisa64r6": ( 8, 0, 0, False, 32),
133 "mipsisa64r6el":( 8, 0, 0, True, 32),
134 },
135 }
136
137 # Add in any extra user supplied data which may come from a BSP layer, removing the
138 # need to always change this class directly
139 extra_machdata = (d and d.getVar("PACKAGEQA_EXTRA_MACHDEFFUNCS" or None) or "").split()
140 for m in extra_machdata:
141 call = m + "(machdata, d)"
142 locs = { "machdata" : machdata, "d" : d}
143 machdata = bb.utils.better_eval(call, locs)
144
145 return machdata
diff --git a/meta-xilinx-core/lib/oe/go.py b/meta-xilinx-core/lib/oe/go.py
new file mode 100644
index 00000000..dfd957d1
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/go.py
@@ -0,0 +1,34 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: MIT
5#
6
7import re
8
9def map_arch(a):
10 if re.match('i.86', a):
11 return '386'
12 elif a == 'x86_64':
13 return 'amd64'
14 elif re.match('arm.*', a):
15 return 'arm'
16 elif re.match('aarch64.*', a):
17 return 'arm64'
18 elif re.match('mips64el.*', a):
19 return 'mips64le'
20 elif re.match('mips64.*', a):
21 return 'mips64'
22 elif a == 'mips':
23 return 'mips'
24 elif a == 'mipsel':
25 return 'mipsle'
26 elif re.match('p(pc|owerpc)(64le)', a):
27 return 'ppc64le'
28 elif re.match('p(pc|owerpc)(64)', a):
29 return 'ppc64'
30 elif a == 'riscv64':
31 return 'riscv64'
32 elif a == 'loongarch64':
33 return 'loong64'
34 return ''
diff --git a/meta-xilinx-core/lib/oe/gpg_sign.py b/meta-xilinx-core/lib/oe/gpg_sign.py
new file mode 100644
index 00000000..ede6186c
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/gpg_sign.py
@@ -0,0 +1,160 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7"""Helper module for GPG signing"""
8
9import bb
10import os
11import shlex
12import subprocess
13import tempfile
14
15class LocalSigner(object):
16 """Class for handling local (on the build host) signing"""
17 def __init__(self, d):
18 self.gpg_bin = d.getVar('GPG_BIN') or \
19 bb.utils.which(os.getenv('PATH'), 'gpg')
20 self.gpg_cmd = [self.gpg_bin]
21 self.gpg_agent_bin = bb.utils.which(os.getenv('PATH'), "gpg-agent")
22 # Without this we see "Cannot allocate memory" errors when running processes in parallel
23 # It needs to be set for any gpg command since any agent launched can stick around in memory
24 # and this parameter must be set.
25 if self.gpg_agent_bin:
26 self.gpg_cmd += ["--agent-program=%s|--auto-expand-secmem" % (self.gpg_agent_bin)]
27 self.gpg_path = d.getVar('GPG_PATH')
28 self.rpm_bin = bb.utils.which(os.getenv('PATH'), "rpmsign")
29 self.gpg_version = self.get_gpg_version()
30
31
32 def export_pubkey(self, output_file, keyid, armor=True):
33 """Export GPG public key to a file"""
34 cmd = self.gpg_cmd + ["--no-permission-warning", "--batch", "--yes", "--export", "-o", output_file]
35 if self.gpg_path:
36 cmd += ["--homedir", self.gpg_path]
37 if armor:
38 cmd += ["--armor"]
39 cmd += [keyid]
40 subprocess.check_output(cmd, stderr=subprocess.STDOUT)
41
42 def sign_rpms(self, files, keyid, passphrase, digest, sign_chunk, fsk=None, fsk_password=None):
43 """Sign RPM files"""
44
45 cmd = self.rpm_bin + " --addsign --define '_gpg_name %s' " % keyid
46 gpg_args = '--no-permission-warning --batch --passphrase=%s --agent-program=%s|--auto-expand-secmem' % (passphrase, self.gpg_agent_bin)
47 if self.gpg_version > (2,1,):
48 gpg_args += ' --pinentry-mode=loopback'
49 cmd += "--define '_gpg_sign_cmd_extra_args %s' " % gpg_args
50 cmd += "--define '_binary_filedigest_algorithm %s' " % digest
51 if self.gpg_bin:
52 cmd += "--define '__gpg %s' " % self.gpg_bin
53 if self.gpg_path:
54 cmd += "--define '_gpg_path %s' " % self.gpg_path
55 if fsk:
56 cmd += "--signfiles --fskpath %s " % fsk
57 if fsk_password:
58 cmd += "--define '_file_signing_key_password %s' " % fsk_password
59
60 # Sign in chunks
61 for i in range(0, len(files), sign_chunk):
62 subprocess.check_output(shlex.split(cmd + ' '.join(files[i:i+sign_chunk])), stderr=subprocess.STDOUT)
63
64 def detach_sign(self, input_file, keyid, passphrase_file, passphrase=None, armor=True, output_suffix=None, use_sha256=False):
65 """Create a detached signature of a file"""
66
67 if passphrase_file and passphrase:
68 raise Exception("You should use either passphrase_file of passphrase, not both")
69
70 cmd = self.gpg_cmd + ['--detach-sign', '--no-permission-warning', '--batch',
71 '--no-tty', '--yes', '--passphrase-fd', '0', '-u', keyid]
72
73 if self.gpg_path:
74 cmd += ['--homedir', self.gpg_path]
75 if armor:
76 cmd += ['--armor']
77 if use_sha256:
78 cmd += ['--digest-algo', "SHA256"]
79
80 #gpg > 2.1 supports password pipes only through the loopback interface
81 #gpg < 2.1 errors out if given unknown parameters
82 if self.gpg_version > (2,1,):
83 cmd += ['--pinentry-mode', 'loopback']
84
85 try:
86 if passphrase_file:
87 with open(passphrase_file) as fobj:
88 passphrase = fobj.readline();
89
90 if not output_suffix:
91 output_suffix = 'asc' if armor else 'sig'
92 output_file = input_file + "." + output_suffix
93 with tempfile.TemporaryDirectory(dir=os.path.dirname(output_file)) as tmp_dir:
94 tmp_file = os.path.join(tmp_dir, os.path.basename(output_file))
95 cmd += ['-o', tmp_file]
96
97 cmd += [input_file]
98
99 job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
100 (_, stderr) = job.communicate(passphrase.encode("utf-8"))
101
102 if job.returncode:
103 bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))
104
105 os.rename(tmp_file, output_file)
106 except IOError as e:
107 bb.error("IO error (%s): %s" % (e.errno, e.strerror))
108 raise Exception("Failed to sign '%s'" % input_file)
109
110 except OSError as e:
111 bb.error("OS error (%s): %s" % (e.errno, e.strerror))
112 raise Exception("Failed to sign '%s" % input_file)
113
114
115 def get_gpg_version(self):
116 """Return the gpg version as a tuple of ints"""
117 try:
118 cmd = self.gpg_cmd + ["--version", "--no-permission-warning"]
119 ver_str = subprocess.check_output(cmd).split()[2].decode("utf-8")
120 return tuple([int(i) for i in ver_str.split("-")[0].split('.')])
121 except subprocess.CalledProcessError as e:
122 bb.fatal("Could not get gpg version: %s" % e)
123
124
125 def verify(self, sig_file, valid_sigs = ''):
126 """Verify signature"""
127 cmd = self.gpg_cmd + ["--verify", "--no-permission-warning", "--status-fd", "1"]
128 if self.gpg_path:
129 cmd += ["--homedir", self.gpg_path]
130
131 cmd += [sig_file]
132 status = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
133 # Valid if any key matches if unspecified
134 if not valid_sigs:
135 ret = False if status.returncode else True
136 return ret
137
138 import re
139 goodsigs = []
140 sigre = re.compile(r'^\[GNUPG:\] GOODSIG (\S+)\s(.*)$')
141 for l in status.stdout.decode("utf-8").splitlines():
142 s = sigre.match(l)
143 if s:
144 goodsigs += [s.group(1)]
145
146 for sig in valid_sigs.split():
147 if sig in goodsigs:
148 return True
149 if len(goodsigs):
150 bb.warn('No accepted signatures found. Good signatures found: %s.' % ' '.join(goodsigs))
151 return False
152
153
154def get_signer(d, backend):
155 """Get signer object for the specified backend"""
156 # Use local signing by default
157 if backend == 'local':
158 return LocalSigner(d)
159 else:
160 bb.fatal("Unsupported signing backend '%s'" % backend)
diff --git a/meta-xilinx-core/lib/oe/license.py b/meta-xilinx-core/lib/oe/license.py
new file mode 100644
index 00000000..d9c8d94d
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/license.py
@@ -0,0 +1,261 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6"""Code for parsing OpenEmbedded license strings"""
7
8import ast
9import re
10from fnmatch import fnmatchcase as fnmatch
11
12def license_ok(license, dont_want_licenses):
13 """ Return False if License exist in dont_want_licenses else True """
14 for dwl in dont_want_licenses:
15 if fnmatch(license, dwl):
16 return False
17 return True
18
19def obsolete_license_list():
20 return ["AGPL-3", "AGPL-3+", "AGPLv3", "AGPLv3+", "AGPLv3.0", "AGPLv3.0+", "AGPL-3.0", "AGPL-3.0+", "BSD-0-Clause",
21 "GPL-1", "GPL-1+", "GPLv1", "GPLv1+", "GPLv1.0", "GPLv1.0+", "GPL-1.0", "GPL-1.0+", "GPL-2", "GPL-2+", "GPLv2",
22 "GPLv2+", "GPLv2.0", "GPLv2.0+", "GPL-2.0", "GPL-2.0+", "GPL-3", "GPL-3+", "GPLv3", "GPLv3+", "GPLv3.0", "GPLv3.0+",
23 "GPL-3.0", "GPL-3.0+", "LGPLv2", "LGPLv2+", "LGPLv2.0", "LGPLv2.0+", "LGPL-2.0", "LGPL-2.0+", "LGPL2.1", "LGPL2.1+",
24 "LGPLv2.1", "LGPLv2.1+", "LGPL-2.1", "LGPL-2.1+", "LGPLv3", "LGPLv3+", "LGPL-3.0", "LGPL-3.0+", "MPL-1", "MPLv1",
25 "MPLv1.1", "MPLv2", "MIT-X", "MIT-style", "openssl", "PSF", "PSFv2", "Python-2", "Apachev2", "Apache-2", "Artisticv1",
26 "Artistic-1", "AFL-2", "AFL-1", "AFLv2", "AFLv1", "CDDLv1", "CDDL-1", "EPLv1.0", "FreeType", "Nauman",
27 "tcl", "vim", "SGIv1"]
28
29class LicenseError(Exception):
30 pass
31
32class LicenseSyntaxError(LicenseError):
33 def __init__(self, licensestr, exc):
34 self.licensestr = licensestr
35 self.exc = exc
36 LicenseError.__init__(self)
37
38 def __str__(self):
39 return "error in '%s': %s" % (self.licensestr, self.exc)
40
41class InvalidLicense(LicenseError):
42 def __init__(self, license):
43 self.license = license
44 LicenseError.__init__(self)
45
46 def __str__(self):
47 return "invalid characters in license '%s'" % self.license
48
49license_operator_chars = '&|() '
50license_operator = re.compile(r'([' + license_operator_chars + '])')
51license_pattern = re.compile(r'[a-zA-Z0-9.+_\-]+$')
52
53class LicenseVisitor(ast.NodeVisitor):
54 """Get elements based on OpenEmbedded license strings"""
55 def get_elements(self, licensestr):
56 new_elements = []
57 elements = list([x for x in license_operator.split(licensestr) if x.strip()])
58 for pos, element in enumerate(elements):
59 if license_pattern.match(element):
60 if pos > 0 and license_pattern.match(elements[pos-1]):
61 new_elements.append('&')
62 element = '"' + element + '"'
63 elif not license_operator.match(element):
64 raise InvalidLicense(element)
65 new_elements.append(element)
66
67 return new_elements
68
69 """Syntax tree visitor which can accept elements previously generated with
70 OpenEmbedded license string"""
71 def visit_elements(self, elements):
72 self.visit(ast.parse(' '.join(elements)))
73
74 """Syntax tree visitor which can accept OpenEmbedded license strings"""
75 def visit_string(self, licensestr):
76 self.visit_elements(self.get_elements(licensestr))
77
78class FlattenVisitor(LicenseVisitor):
79 """Flatten a license tree (parsed from a string) by selecting one of each
80 set of OR options, in the way the user specifies"""
81 def __init__(self, choose_licenses):
82 self.choose_licenses = choose_licenses
83 self.licenses = []
84 LicenseVisitor.__init__(self)
85
86 def visit_Str(self, node):
87 self.licenses.append(node.s)
88
89 def visit_Constant(self, node):
90 self.licenses.append(node.value)
91
92 def visit_BinOp(self, node):
93 if isinstance(node.op, ast.BitOr):
94 left = FlattenVisitor(self.choose_licenses)
95 left.visit(node.left)
96
97 right = FlattenVisitor(self.choose_licenses)
98 right.visit(node.right)
99
100 selected = self.choose_licenses(left.licenses, right.licenses)
101 self.licenses.extend(selected)
102 else:
103 self.generic_visit(node)
104
105def flattened_licenses(licensestr, choose_licenses):
106 """Given a license string and choose_licenses function, return a flat list of licenses"""
107 flatten = FlattenVisitor(choose_licenses)
108 try:
109 flatten.visit_string(licensestr)
110 except SyntaxError as exc:
111 raise LicenseSyntaxError(licensestr, exc)
112 return flatten.licenses
113
114def is_included(licensestr, include_licenses=None, exclude_licenses=None):
115 """Given a license string, a list of licenses to include and a list of
116 licenses to exclude, determine if the license string matches the include
117 list and does not match the exclude list.
118
119 Returns a tuple holding the boolean state and a list of the applicable
120 licenses that were excluded if state is False, or the licenses that were
121 included if the state is True."""
122
123 def include_license(license):
124 return any(fnmatch(license, pattern) for pattern in include_licenses)
125
126 def exclude_license(license):
127 return any(fnmatch(license, pattern) for pattern in exclude_licenses)
128
129 def choose_licenses(alpha, beta):
130 """Select the option in an OR which is the 'best' (has the most
131 included licenses and no excluded licenses)."""
132 # The factor 1000 below is arbitrary, just expected to be much larger
133 # than the number of licenses actually specified. That way the weight
134 # will be negative if the list of licenses contains an excluded license,
135 # but still gives a higher weight to the list with the most included
136 # licenses.
137 alpha_weight = (len(list(filter(include_license, alpha))) -
138 1000 * (len(list(filter(exclude_license, alpha))) > 0))
139 beta_weight = (len(list(filter(include_license, beta))) -
140 1000 * (len(list(filter(exclude_license, beta))) > 0))
141 if alpha_weight >= beta_weight:
142 return alpha
143 else:
144 return beta
145
146 if not include_licenses:
147 include_licenses = ['*']
148
149 if not exclude_licenses:
150 exclude_licenses = []
151
152 licenses = flattened_licenses(licensestr, choose_licenses)
153 excluded = [lic for lic in licenses if exclude_license(lic)]
154 included = [lic for lic in licenses if include_license(lic)]
155 if excluded:
156 return False, excluded
157 else:
158 return True, included
159
160class ManifestVisitor(LicenseVisitor):
161 """Walk license tree (parsed from a string) removing the incompatible
162 licenses specified"""
163 def __init__(self, dont_want_licenses, canonical_license, d):
164 self._dont_want_licenses = dont_want_licenses
165 self._canonical_license = canonical_license
166 self._d = d
167 self._operators = []
168
169 self.licenses = []
170 self.licensestr = ''
171
172 LicenseVisitor.__init__(self)
173
174 def visit(self, node):
175 if isinstance(node, ast.Str):
176 lic = node.s
177
178 if license_ok(self._canonical_license(self._d, lic),
179 self._dont_want_licenses) == True:
180 if self._operators:
181 ops = []
182 for op in self._operators:
183 if op == '[':
184 ops.append(op)
185 elif op == ']':
186 ops.append(op)
187 else:
188 if not ops:
189 ops.append(op)
190 elif ops[-1] in ['[', ']']:
191 ops.append(op)
192 else:
193 ops[-1] = op
194
195 for op in ops:
196 if op == '[' or op == ']':
197 self.licensestr += op
198 elif self.licenses:
199 self.licensestr += ' ' + op + ' '
200
201 self._operators = []
202
203 self.licensestr += lic
204 self.licenses.append(lic)
205 elif isinstance(node, ast.BitAnd):
206 self._operators.append("&")
207 elif isinstance(node, ast.BitOr):
208 self._operators.append("|")
209 elif isinstance(node, ast.List):
210 self._operators.append("[")
211 elif isinstance(node, ast.Load):
212 self.licensestr += "]"
213
214 self.generic_visit(node)
215
216def manifest_licenses(licensestr, dont_want_licenses, canonical_license, d):
217 """Given a license string and dont_want_licenses list,
218 return license string filtered and a list of licenses"""
219 manifest = ManifestVisitor(dont_want_licenses, canonical_license, d)
220
221 try:
222 elements = manifest.get_elements(licensestr)
223
224 # Replace '()' to '[]' for handle in ast as List and Load types.
225 elements = ['[' if e == '(' else e for e in elements]
226 elements = [']' if e == ')' else e for e in elements]
227
228 manifest.visit_elements(elements)
229 except SyntaxError as exc:
230 raise LicenseSyntaxError(licensestr, exc)
231
232 # Replace '[]' to '()' for output correct license.
233 manifest.licensestr = manifest.licensestr.replace('[', '(').replace(']', ')')
234
235 return (manifest.licensestr, manifest.licenses)
236
237class ListVisitor(LicenseVisitor):
238 """Record all different licenses found in the license string"""
239 def __init__(self):
240 self.licenses = set()
241
242 def visit_Str(self, node):
243 self.licenses.add(node.s)
244
245 def visit_Constant(self, node):
246 self.licenses.add(node.value)
247
248def list_licenses(licensestr):
249 """Simply get a list of all licenses mentioned in a license string.
250 Binary operators are not applied or taken into account in any way"""
251 visitor = ListVisitor()
252 try:
253 visitor.visit_string(licensestr)
254 except SyntaxError as exc:
255 raise LicenseSyntaxError(licensestr, exc)
256 return visitor.licenses
257
258def apply_pkg_license_exception(pkg, bad_licenses, exceptions):
259 """Return remaining bad licenses after removing any package exceptions"""
260
261 return [lic for lic in bad_licenses if pkg + ':' + lic not in exceptions]
diff --git a/meta-xilinx-core/lib/oe/lsb.py b/meta-xilinx-core/lib/oe/lsb.py
new file mode 100644
index 00000000..3ec03e50
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/lsb.py
@@ -0,0 +1,123 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7def get_os_release():
8 """Get all key-value pairs from /etc/os-release as a dict"""
9 from collections import OrderedDict
10
11 data = OrderedDict()
12 if os.path.exists('/etc/os-release'):
13 with open('/etc/os-release') as f:
14 for line in f:
15 try:
16 key, val = line.rstrip().split('=', 1)
17 except ValueError:
18 continue
19 data[key.strip()] = val.strip('"')
20 return data
21
22def release_dict_osr():
23 """ Populate a dict with pertinent values from /etc/os-release """
24 data = {}
25 os_release = get_os_release()
26 if 'ID' in os_release:
27 data['DISTRIB_ID'] = os_release['ID']
28 if 'VERSION_ID' in os_release:
29 data['DISTRIB_RELEASE'] = os_release['VERSION_ID']
30
31 return data
32
33def release_dict_lsb():
34 """ Return the output of lsb_release -ir as a dictionary """
35 from subprocess import PIPE
36
37 try:
38 output, err = bb.process.run(['lsb_release', '-ir'], stderr=PIPE)
39 except bb.process.CmdError as exc:
40 return {}
41
42 lsb_map = { 'Distributor ID': 'DISTRIB_ID',
43 'Release': 'DISTRIB_RELEASE'}
44 lsb_keys = lsb_map.keys()
45
46 data = {}
47 for line in output.splitlines():
48 if line.startswith("-e"):
49 line = line[3:]
50 try:
51 key, value = line.split(":\t", 1)
52 except ValueError:
53 continue
54 if key in lsb_keys:
55 data[lsb_map[key]] = value
56
57 if len(data.keys()) != 2:
58 return None
59
60 return data
61
62def release_dict_file():
63 """ Try to gather release information manually when other methods fail """
64 data = {}
65 try:
66 if os.path.exists('/etc/lsb-release'):
67 data = {}
68 with open('/etc/lsb-release') as f:
69 for line in f:
70 key, value = line.split("=", 1)
71 data[key] = value.strip()
72 elif os.path.exists('/etc/redhat-release'):
73 data = {}
74 with open('/etc/redhat-release') as f:
75 distro = f.readline().strip()
76 import re
77 match = re.match(r'(.*) release (.*) \((.*)\)', distro)
78 if match:
79 data['DISTRIB_ID'] = match.group(1)
80 data['DISTRIB_RELEASE'] = match.group(2)
81 elif os.path.exists('/etc/SuSE-release'):
82 data = {}
83 data['DISTRIB_ID'] = 'SUSE LINUX'
84 with open('/etc/SuSE-release') as f:
85 for line in f:
86 if line.startswith('VERSION = '):
87 data['DISTRIB_RELEASE'] = line[10:].rstrip()
88 break
89
90 except IOError:
91 return {}
92 return data
93
94def distro_identifier(adjust_hook=None):
95 """Return a distro identifier string based upon lsb_release -ri,
96 with optional adjustment via a hook"""
97
98 import re
99
100 # Try /etc/os-release first, then the output of `lsb_release -ir` and
101 # finally fall back on parsing various release files in order to determine
102 # host distro name and version.
103 distro_data = release_dict_osr()
104 if not distro_data:
105 distro_data = release_dict_lsb()
106 if not distro_data:
107 distro_data = release_dict_file()
108
109 distro_id = distro_data.get('DISTRIB_ID', '')
110 release = distro_data.get('DISTRIB_RELEASE', '')
111
112 if adjust_hook:
113 distro_id, release = adjust_hook(distro_id, release)
114 if not distro_id:
115 return "unknown"
116 # Filter out any non-alphanumerics and convert to lowercase
117 distro_id = re.sub(r'\W', '', distro_id).lower()
118
119 if release:
120 id_str = '{0}-{1}'.format(distro_id, release)
121 else:
122 id_str = distro_id
123 return id_str.replace(' ','-').replace('/','-')
diff --git a/meta-xilinx-core/lib/oe/maketype.py b/meta-xilinx-core/lib/oe/maketype.py
new file mode 100644
index 00000000..7a83bdf6
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/maketype.py
@@ -0,0 +1,107 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6"""OpenEmbedded variable typing support
7
8Types are defined in the metadata by name, using the 'type' flag on a
9variable. Other flags may be utilized in the construction of the types. See
10the arguments of the type's factory for details.
11"""
12
13import inspect
14import oe.types as types
15from collections.abc import Callable
16
17available_types = {}
18
19class MissingFlag(TypeError):
20 """A particular flag is required to construct the type, but has not been
21 provided."""
22 def __init__(self, flag, type):
23 self.flag = flag
24 self.type = type
25 TypeError.__init__(self)
26
27 def __str__(self):
28 return "Type '%s' requires flag '%s'" % (self.type, self.flag)
29
30def factory(var_type):
31 """Return the factory for a specified type."""
32 if var_type is None:
33 raise TypeError("No type specified. Valid types: %s" %
34 ', '.join(available_types))
35 try:
36 return available_types[var_type]
37 except KeyError:
38 raise TypeError("Invalid type '%s':\n Valid types: %s" %
39 (var_type, ', '.join(available_types)))
40
41def create(value, var_type, **flags):
42 """Create an object of the specified type, given the specified flags and
43 string value."""
44 obj = factory(var_type)
45 objflags = {}
46 for flag in obj.flags:
47 if flag not in flags:
48 if flag not in obj.optflags:
49 raise MissingFlag(flag, var_type)
50 else:
51 objflags[flag] = flags[flag]
52
53 return obj(value, **objflags)
54
55def get_callable_args(obj):
56 """Grab all but the first argument of the specified callable, returning
57 the list, as well as a list of which of the arguments have default
58 values."""
59 if type(obj) is type:
60 obj = obj.__init__
61
62 sig = inspect.signature(obj)
63 args = list(sig.parameters.keys())
64 defaults = list(s for s in sig.parameters.keys() if sig.parameters[s].default != inspect.Parameter.empty)
65 flaglist = []
66 if args:
67 if len(args) > 1 and args[0] == 'self':
68 args = args[1:]
69 flaglist.extend(args)
70
71 optional = set()
72 if defaults:
73 optional |= set(flaglist[-len(defaults):])
74 return flaglist, optional
75
76def factory_setup(name, obj):
77 """Prepare a factory for use."""
78 args, optional = get_callable_args(obj)
79 extra_args = args[1:]
80 if extra_args:
81 obj.flags, optional = extra_args, optional
82 obj.optflags = set(optional)
83 else:
84 obj.flags = obj.optflags = ()
85
86 if not hasattr(obj, 'name'):
87 obj.name = name
88
89def register(name, factory):
90 """Register a type, given its name and a factory callable.
91
92 Determines the required and optional flags from the factory's
93 arguments."""
94 factory_setup(name, factory)
95 available_types[factory.name] = factory
96
97
98# Register all our included types
99for name in dir(types):
100 if name.startswith('_'):
101 continue
102
103 obj = getattr(types, name)
104 if not isinstance(obj, Callable):
105 continue
106
107 register(name, obj)
diff --git a/meta-xilinx-core/lib/oe/manifest.py b/meta-xilinx-core/lib/oe/manifest.py
new file mode 100644
index 00000000..61f18adc
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/manifest.py
@@ -0,0 +1,206 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7from abc import ABCMeta, abstractmethod
8import os
9import re
10import bb
11
12class Manifest(object, metaclass=ABCMeta):
13 """
14 This is an abstract class. Do not instantiate this directly.
15 """
16
17 PKG_TYPE_MUST_INSTALL = "mip"
18 PKG_TYPE_MULTILIB = "mlp"
19 PKG_TYPE_LANGUAGE = "lgp"
20 PKG_TYPE_ATTEMPT_ONLY = "aop"
21
22 MANIFEST_TYPE_IMAGE = "image"
23 MANIFEST_TYPE_SDK_HOST = "sdk_host"
24 MANIFEST_TYPE_SDK_TARGET = "sdk_target"
25
26 var_maps = {
27 MANIFEST_TYPE_IMAGE: {
28 "PACKAGE_INSTALL": PKG_TYPE_MUST_INSTALL,
29 "PACKAGE_INSTALL_ATTEMPTONLY": PKG_TYPE_ATTEMPT_ONLY,
30 "LINGUAS_INSTALL": PKG_TYPE_LANGUAGE
31 },
32 MANIFEST_TYPE_SDK_HOST: {
33 "TOOLCHAIN_HOST_TASK": PKG_TYPE_MUST_INSTALL,
34 "TOOLCHAIN_HOST_TASK_ATTEMPTONLY": PKG_TYPE_ATTEMPT_ONLY
35 },
36 MANIFEST_TYPE_SDK_TARGET: {
37 "TOOLCHAIN_TARGET_TASK": PKG_TYPE_MUST_INSTALL,
38 "TOOLCHAIN_TARGET_TASK_ATTEMPTONLY": PKG_TYPE_ATTEMPT_ONLY
39 }
40 }
41
42 INSTALL_ORDER = [
43 PKG_TYPE_LANGUAGE,
44 PKG_TYPE_MUST_INSTALL,
45 PKG_TYPE_ATTEMPT_ONLY,
46 PKG_TYPE_MULTILIB
47 ]
48
49 initial_manifest_file_header = \
50 "# This file was generated automatically and contains the packages\n" \
51 "# passed on to the package manager in order to create the rootfs.\n\n" \
52 "# Format:\n" \
53 "# <package_type>,<package_name>\n" \
54 "# where:\n" \
55 "# <package_type> can be:\n" \
56 "# 'mip' = must install package\n" \
57 "# 'aop' = attempt only package\n" \
58 "# 'mlp' = multilib package\n" \
59 "# 'lgp' = language package\n\n"
60
61 def __init__(self, d, manifest_dir=None, manifest_type=MANIFEST_TYPE_IMAGE):
62 self.d = d
63 self.manifest_type = manifest_type
64
65 if manifest_dir is None:
66 if manifest_type != self.MANIFEST_TYPE_IMAGE:
67 self.manifest_dir = self.d.getVar('SDK_DIR')
68 else:
69 self.manifest_dir = self.d.getVar('WORKDIR')
70 else:
71 self.manifest_dir = manifest_dir
72
73 bb.utils.mkdirhier(self.manifest_dir)
74
75 self.initial_manifest = os.path.join(self.manifest_dir, "%s_initial_manifest" % manifest_type)
76 self.final_manifest = os.path.join(self.manifest_dir, "%s_final_manifest" % manifest_type)
77 self.full_manifest = os.path.join(self.manifest_dir, "%s_full_manifest" % manifest_type)
78
79 # packages in the following vars will be split in 'must install' and
80 # 'multilib'
81 self.vars_to_split = ["PACKAGE_INSTALL",
82 "TOOLCHAIN_HOST_TASK",
83 "TOOLCHAIN_TARGET_TASK"]
84
85 """
86 This creates a standard initial manifest for core-image-(minimal|sato|sato-sdk).
87 This will be used for testing until the class is implemented properly!
88 """
89 def _create_dummy_initial(self):
90 image_rootfs = self.d.getVar('IMAGE_ROOTFS')
91 pkg_list = dict()
92 if image_rootfs.find("core-image-sato-sdk") > 0:
93 pkg_list[self.PKG_TYPE_MUST_INSTALL] = \
94 "packagegroup-core-x11-sato-games packagegroup-base-extended " \
95 "packagegroup-core-x11-sato packagegroup-core-x11-base " \
96 "packagegroup-core-sdk packagegroup-core-tools-debug " \
97 "packagegroup-core-boot packagegroup-core-tools-testapps " \
98 "packagegroup-core-eclipse-debug packagegroup-core-qt-demoapps " \
99 "apt packagegroup-core-tools-profile psplash " \
100 "packagegroup-core-standalone-sdk-target " \
101 "packagegroup-core-ssh-openssh dpkg kernel-dev"
102 pkg_list[self.PKG_TYPE_LANGUAGE] = \
103 "locale-base-en-us locale-base-en-gb"
104 elif image_rootfs.find("core-image-sato") > 0:
105 pkg_list[self.PKG_TYPE_MUST_INSTALL] = \
106 "packagegroup-core-ssh-dropbear packagegroup-core-x11-sato-games " \
107 "packagegroup-core-x11-base psplash apt dpkg packagegroup-base-extended " \
108 "packagegroup-core-x11-sato packagegroup-core-boot"
109 pkg_list['lgp'] = \
110 "locale-base-en-us locale-base-en-gb"
111 elif image_rootfs.find("core-image-minimal") > 0:
112 pkg_list[self.PKG_TYPE_MUST_INSTALL] = "packagegroup-core-boot"
113
114 with open(self.initial_manifest, "w+") as manifest:
115 manifest.write(self.initial_manifest_file_header)
116
117 for pkg_type in pkg_list:
118 for pkg in pkg_list[pkg_type].split():
119 manifest.write("%s,%s\n" % (pkg_type, pkg))
120
121 """
122 This will create the initial manifest which will be used by Rootfs class to
123 generate the rootfs
124 """
125 @abstractmethod
126 def create_initial(self):
127 pass
128
129 """
130 This creates the manifest after everything has been installed.
131 """
132 @abstractmethod
133 def create_final(self):
134 pass
135
136 """
137 This creates the manifest after the package in initial manifest has been
138 dummy installed. It lists all *to be installed* packages. There is no real
139 installation, just a test.
140 """
141 @abstractmethod
142 def create_full(self, pm):
143 pass
144
145 """
146 The following function parses an initial manifest and returns a dictionary
147 object with the must install, attempt only, multilib and language packages.
148 """
149 def parse_initial_manifest(self):
150 pkgs = dict()
151
152 with open(self.initial_manifest) as manifest:
153 for line in manifest.read().split('\n'):
154 comment = re.match("^#.*", line)
155 pattern = "^(%s|%s|%s|%s),(.*)$" % \
156 (self.PKG_TYPE_MUST_INSTALL,
157 self.PKG_TYPE_ATTEMPT_ONLY,
158 self.PKG_TYPE_MULTILIB,
159 self.PKG_TYPE_LANGUAGE)
160 pkg = re.match(pattern, line)
161
162 if comment is not None:
163 continue
164
165 if pkg is not None:
166 pkg_type = pkg.group(1)
167 pkg_name = pkg.group(2)
168
169 if not pkg_type in pkgs:
170 pkgs[pkg_type] = [pkg_name]
171 else:
172 pkgs[pkg_type].append(pkg_name)
173
174 return pkgs
175
176 '''
177 This following function parses a full manifest and return a list
178 object with packages.
179 '''
180 def parse_full_manifest(self):
181 installed_pkgs = list()
182 if not os.path.exists(self.full_manifest):
183 bb.note('full manifest not exist')
184 return installed_pkgs
185
186 with open(self.full_manifest, 'r') as manifest:
187 for pkg in manifest.read().split('\n'):
188 installed_pkgs.append(pkg.strip())
189
190 return installed_pkgs
191
192
193
194def create_manifest(d, final_manifest=False, manifest_dir=None,
195 manifest_type=Manifest.MANIFEST_TYPE_IMAGE):
196 import importlib
197 manifest = importlib.import_module('oe.package_manager.' + d.getVar('IMAGE_PKGTYPE') + '.manifest').PkgManifest(d, manifest_dir, manifest_type)
198
199 if final_manifest:
200 manifest.create_final()
201 else:
202 manifest.create_initial()
203
204
205if __name__ == "__main__":
206 pass
diff --git a/meta-xilinx-core/lib/oe/npm_registry.py b/meta-xilinx-core/lib/oe/npm_registry.py
new file mode 100644
index 00000000..d97ced7c
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/npm_registry.py
@@ -0,0 +1,175 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: MIT
5#
6
7import bb
8import json
9import subprocess
10
11_ALWAYS_SAFE = frozenset('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
12 'abcdefghijklmnopqrstuvwxyz'
13 '0123456789'
14 '_.-~()')
15
16MISSING_OK = object()
17
18REGISTRY = "https://registry.npmjs.org"
19
20# we can not use urllib.parse here because npm expects lowercase
21# hex-chars but urllib generates uppercase ones
22def uri_quote(s, safe = '/'):
23 res = ""
24 safe_set = set(safe)
25 for c in s:
26 if c in _ALWAYS_SAFE or c in safe_set:
27 res += c
28 else:
29 res += '%%%02x' % ord(c)
30 return res
31
32class PackageJson:
33 def __init__(self, spec):
34 self.__spec = spec
35
36 @property
37 def name(self):
38 return self.__spec['name']
39
40 @property
41 def version(self):
42 return self.__spec['version']
43
44 @property
45 def empty_manifest(self):
46 return {
47 'name': self.name,
48 'description': self.__spec.get('description', ''),
49 'versions': {},
50 }
51
52 def base_filename(self):
53 return uri_quote(self.name, safe = '@')
54
55 def as_manifest_entry(self, tarball_uri):
56 res = {}
57
58 ## NOTE: 'npm install' requires more than basic meta information;
59 ## e.g. it takes 'bin' from this manifest entry but not the actual
60 ## 'package.json'
61 for (idx,dflt) in [('name', None),
62 ('description', ""),
63 ('version', None),
64 ('bin', MISSING_OK),
65 ('man', MISSING_OK),
66 ('scripts', MISSING_OK),
67 ('directories', MISSING_OK),
68 ('dependencies', MISSING_OK),
69 ('devDependencies', MISSING_OK),
70 ('optionalDependencies', MISSING_OK),
71 ('license', "unknown")]:
72 if idx in self.__spec:
73 res[idx] = self.__spec[idx]
74 elif dflt == MISSING_OK:
75 pass
76 elif dflt != None:
77 res[idx] = dflt
78 else:
79 raise Exception("%s-%s: missing key %s" % (self.name,
80 self.version,
81 idx))
82
83 res['dist'] = {
84 'tarball': tarball_uri,
85 }
86
87 return res
88
89class ManifestImpl:
90 def __init__(self, base_fname, spec):
91 self.__base = base_fname
92 self.__spec = spec
93
94 def load(self):
95 try:
96 with open(self.filename, "r") as f:
97 res = json.load(f)
98 except IOError:
99 res = self.__spec.empty_manifest
100
101 return res
102
103 def save(self, meta):
104 with open(self.filename, "w") as f:
105 json.dump(meta, f, indent = 2)
106
107 @property
108 def filename(self):
109 return self.__base + ".meta"
110
111class Manifest:
112 def __init__(self, base_fname, spec):
113 self.__base = base_fname
114 self.__spec = spec
115 self.__lockf = None
116 self.__impl = None
117
118 def __enter__(self):
119 self.__lockf = bb.utils.lockfile(self.__base + ".lock")
120 self.__impl = ManifestImpl(self.__base, self.__spec)
121 return self.__impl
122
123 def __exit__(self, exc_type, exc_val, exc_tb):
124 bb.utils.unlockfile(self.__lockf)
125
126class NpmCache:
127 def __init__(self, cache):
128 self.__cache = cache
129
130 @property
131 def path(self):
132 return self.__cache
133
134 def run(self, type, key, fname):
135 subprocess.run(['oe-npm-cache', self.__cache, type, key, fname],
136 check = True)
137
138class NpmRegistry:
139 def __init__(self, path, cache):
140 self.__path = path
141 self.__cache = NpmCache(cache + '/_cacache')
142 bb.utils.mkdirhier(self.__path)
143 bb.utils.mkdirhier(self.__cache.path)
144
145 @staticmethod
146 ## This function is critical and must match nodejs expectations
147 def _meta_uri(spec):
148 return REGISTRY + '/' + uri_quote(spec.name, safe = '@')
149
150 @staticmethod
151 ## Exact return value does not matter; just make it look like a
152 ## usual registry url
153 def _tarball_uri(spec):
154 return '%s/%s/-/%s-%s.tgz' % (REGISTRY,
155 uri_quote(spec.name, safe = '@'),
156 uri_quote(spec.name, safe = '@/'),
157 spec.version)
158
159 def add_pkg(self, tarball, pkg_json):
160 pkg_json = PackageJson(pkg_json)
161 base = os.path.join(self.__path, pkg_json.base_filename())
162
163 with Manifest(base, pkg_json) as manifest:
164 meta = manifest.load()
165 tarball_uri = self._tarball_uri(pkg_json)
166
167 meta['versions'][pkg_json.version] = pkg_json.as_manifest_entry(tarball_uri)
168
169 manifest.save(meta)
170
171 ## Cache entries are a little bit dependent on the nodejs
172 ## version; version specific cache implementation must
173 ## mitigate differences
174 self.__cache.run('meta', self._meta_uri(pkg_json), manifest.filename);
175 self.__cache.run('tgz', tarball_uri, tarball);
diff --git a/meta-xilinx-core/lib/oe/overlayfs.py b/meta-xilinx-core/lib/oe/overlayfs.py
new file mode 100644
index 00000000..8b88900f
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/overlayfs.py
@@ -0,0 +1,54 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6# This file contains common functions for overlayfs and its QA check
7
8# this function is based on https://github.com/systemd/systemd/blob/main/src/basic/unit-name.c
9def escapeSystemdUnitName(path):
10 escapeMap = {
11 '/': '-',
12 '-': "\\x2d",
13 '\\': "\\x5d"
14 }
15 return "".join([escapeMap.get(c, c) for c in path.strip('/')])
16
17def strForBash(s):
18 return s.replace('\\', '\\\\')
19
20def allOverlaysUnitName(d):
21 return d.getVar('PN') + '-overlays.service'
22
23def mountUnitName(unit):
24 return escapeSystemdUnitName(unit) + '.mount'
25
26def helperUnitName(unit):
27 return escapeSystemdUnitName(unit) + '-create-upper-dir.service'
28
29def unitFileList(d):
30 fileList = []
31 overlayMountPoints = d.getVarFlags("OVERLAYFS_MOUNT_POINT")
32
33 if not overlayMountPoints:
34 bb.fatal("A recipe uses overlayfs class but there is no OVERLAYFS_MOUNT_POINT set in your MACHINE configuration")
35
36 # check that we have required mount points set first
37 requiredMountPoints = d.getVarFlags('OVERLAYFS_WRITABLE_PATHS')
38 for mountPoint in requiredMountPoints:
39 if mountPoint not in overlayMountPoints:
40 bb.fatal("Missing required mount point for OVERLAYFS_MOUNT_POINT[%s] in your MACHINE configuration" % mountPoint)
41
42 for mountPoint in overlayMountPoints:
43 mountPointList = d.getVarFlag('OVERLAYFS_WRITABLE_PATHS', mountPoint)
44 if not mountPointList:
45 bb.debug(1, "No mount points defined for %s flag, don't add to file list", mountPoint)
46 continue
47 for path in mountPointList.split():
48 fileList.append(mountUnitName(path))
49 fileList.append(helperUnitName(path))
50
51 fileList.append(allOverlaysUnitName(d))
52
53 return fileList
54
diff --git a/meta-xilinx-core/lib/oe/package.py b/meta-xilinx-core/lib/oe/package.py
new file mode 100644
index 00000000..af0923a6
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/package.py
@@ -0,0 +1,2065 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import errno
8import fnmatch
9import itertools
10import os
11import shlex
12import re
13import glob
14import stat
15import mmap
16import subprocess
17import shutil
18
19import oe.cachedpath
20
21def runstrip(arg):
22 # Function to strip a single file, called from split_and_strip_files below
23 # A working 'file' (one which works on the target architecture)
24 #
25 # The elftype is a bit pattern (explained in is_elf below) to tell
26 # us what type of file we're processing...
27 # 4 - executable
28 # 8 - shared library
29 # 16 - kernel module
30
31 if len(arg) == 3:
32 (file, elftype, strip) = arg
33 extra_strip_sections = ''
34 else:
35 (file, elftype, strip, extra_strip_sections) = arg
36
37 newmode = None
38 if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
39 origmode = os.stat(file)[stat.ST_MODE]
40 newmode = origmode | stat.S_IWRITE | stat.S_IREAD
41 os.chmod(file, newmode)
42
43 stripcmd = [strip]
44 skip_strip = False
45 # kernel module
46 if elftype & 16:
47 if is_kernel_module_signed(file):
48 bb.debug(1, "Skip strip on signed module %s" % file)
49 skip_strip = True
50 else:
51 stripcmd.extend(["--strip-debug", "--remove-section=.comment",
52 "--remove-section=.note", "--preserve-dates"])
53 # .so and shared library
54 elif ".so" in file and elftype & 8:
55 stripcmd.extend(["--remove-section=.comment", "--remove-section=.note", "--strip-unneeded"])
56 # shared or executable:
57 elif elftype & 8 or elftype & 4:
58 stripcmd.extend(["--remove-section=.comment", "--remove-section=.note"])
59 if extra_strip_sections != '':
60 for section in extra_strip_sections.split():
61 stripcmd.extend(["--remove-section=" + section])
62
63 stripcmd.append(file)
64 bb.debug(1, "runstrip: %s" % stripcmd)
65
66 if not skip_strip:
67 output = subprocess.check_output(stripcmd, stderr=subprocess.STDOUT)
68
69 if newmode:
70 os.chmod(file, origmode)
71
72# Detect .ko module by searching for "vermagic=" string
73def is_kernel_module(path):
74 with open(path) as f:
75 return mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ).find(b"vermagic=") >= 0
76
77# Detect if .ko module is signed
78def is_kernel_module_signed(path):
79 with open(path, "rb") as f:
80 f.seek(-28, 2)
81 module_tail = f.read()
82 return "Module signature appended" in "".join(chr(c) for c in bytearray(module_tail))
83
84# Return type (bits):
85# 0 - not elf
86# 1 - ELF
87# 2 - stripped
88# 4 - executable
89# 8 - shared library
90# 16 - kernel module
91def is_elf(path):
92 exec_type = 0
93 result = subprocess.check_output(["file", "-b", path], stderr=subprocess.STDOUT).decode("utf-8")
94
95 if "ELF" in result:
96 exec_type |= 1
97 if "not stripped" not in result:
98 exec_type |= 2
99 if "executable" in result:
100 exec_type |= 4
101 if "shared" in result:
102 exec_type |= 8
103 if "relocatable" in result:
104 if path.endswith(".ko") and path.find("/lib/modules/") != -1 and is_kernel_module(path):
105 exec_type |= 16
106 return (path, exec_type)
107
108def is_static_lib(path):
109 if path.endswith('.a') and not os.path.islink(path):
110 with open(path, 'rb') as fh:
111 # The magic must include the first slash to avoid
112 # matching golang static libraries
113 magic = b'!<arch>\x0a/'
114 start = fh.read(len(magic))
115 return start == magic
116 return False
117
118def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, max_process, qa_already_stripped=False):
119 """
120 Strip executable code (like executables, shared libraries) _in_place_
121 - Based on sysroot_strip in staging.bbclass
122 :param dstdir: directory in which to strip files
123 :param strip_cmd: Strip command (usually ${STRIP})
124 :param libdir: ${libdir} - strip .so files in this directory
125 :param base_libdir: ${base_libdir} - strip .so files in this directory
126 :param max_process: number of stripping processes started in parallel
127 :param qa_already_stripped: Set to True if already-stripped' in ${INSANE_SKIP}
128 This is for proper logging and messages only.
129 """
130 import stat, errno, oe.path, oe.utils
131
132 elffiles = {}
133 inodes = {}
134 libdir = os.path.abspath(dstdir + os.sep + libdir)
135 base_libdir = os.path.abspath(dstdir + os.sep + base_libdir)
136 exec_mask = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
137 #
138 # First lets figure out all of the files we may have to process
139 #
140 checkelf = []
141 inodecache = {}
142 for root, dirs, files in os.walk(dstdir):
143 for f in files:
144 file = os.path.join(root, f)
145
146 try:
147 ltarget = oe.path.realpath(file, dstdir, False)
148 s = os.lstat(ltarget)
149 except OSError as e:
150 (err, strerror) = e.args
151 if err != errno.ENOENT:
152 raise
153 # Skip broken symlinks
154 continue
155 if not s:
156 continue
157 # Check its an excutable
158 if s[stat.ST_MODE] & exec_mask \
159 or ((file.startswith(libdir) or file.startswith(base_libdir)) and ".so" in f) \
160 or file.endswith('.ko'):
161 # If it's a symlink, and points to an ELF file, we capture the readlink target
162 if os.path.islink(file):
163 continue
164
165 # It's a file (or hardlink), not a link
166 # ...but is it ELF, and is it already stripped?
167 checkelf.append(file)
168 inodecache[file] = s.st_ino
169 results = oe.utils.multiprocess_launch_mp(is_elf, checkelf, max_process)
170 for (file, elf_file) in results:
171 #elf_file = is_elf(file)
172 if elf_file & 1:
173 if elf_file & 2:
174 if qa_already_stripped:
175 bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dstdir):], pn))
176 else:
177 bb.warn("File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dstdir):], pn))
178 continue
179
180 if inodecache[file] in inodes:
181 os.unlink(file)
182 os.link(inodes[inodecache[file]], file)
183 else:
184 # break hardlinks so that we do not strip the original.
185 inodes[inodecache[file]] = file
186 bb.utils.break_hardlinks(file)
187 elffiles[file] = elf_file
188
189 #
190 # Now strip them (in parallel)
191 #
192 sfiles = []
193 for file in elffiles:
194 elf_file = int(elffiles[file])
195 sfiles.append((file, elf_file, strip_cmd))
196
197 oe.utils.multiprocess_launch_mp(runstrip, sfiles, max_process)
198
199
200def file_translate(file):
201 ft = file.replace("@", "@at@")
202 ft = ft.replace(" ", "@space@")
203 ft = ft.replace("\t", "@tab@")
204 ft = ft.replace("[", "@openbrace@")
205 ft = ft.replace("]", "@closebrace@")
206 ft = ft.replace("_", "@underscore@")
207 return ft
208
209def filedeprunner(arg):
210 import re, subprocess, shlex
211
212 (pkg, pkgfiles, rpmdeps, pkgdest) = arg
213 provides = {}
214 requires = {}
215
216 file_re = re.compile(r'\s+\d+\s(.*)')
217 dep_re = re.compile(r'\s+(\S)\s+(.*)')
218 r = re.compile(r'[<>=]+\s+\S*')
219
220 def process_deps(pipe, pkg, pkgdest, provides, requires):
221 file = None
222 for line in pipe.split("\n"):
223
224 m = file_re.match(line)
225 if m:
226 file = m.group(1)
227 file = file.replace(pkgdest + "/" + pkg, "")
228 file = file_translate(file)
229 continue
230
231 m = dep_re.match(line)
232 if not m or not file:
233 continue
234
235 type, dep = m.groups()
236
237 if type == 'R':
238 i = requires
239 elif type == 'P':
240 i = provides
241 else:
242 continue
243
244 if dep.startswith("python("):
245 continue
246
247 # Ignore all perl(VMS::...) and perl(Mac::...) dependencies. These
248 # are typically used conditionally from the Perl code, but are
249 # generated as unconditional dependencies.
250 if dep.startswith('perl(VMS::') or dep.startswith('perl(Mac::'):
251 continue
252
253 # Ignore perl dependencies on .pl files.
254 if dep.startswith('perl(') and dep.endswith('.pl)'):
255 continue
256
257 # Remove perl versions and perl module versions since they typically
258 # do not make sense when used as package versions.
259 if dep.startswith('perl') and r.search(dep):
260 dep = dep.split()[0]
261
262 # Put parentheses around any version specifications.
263 dep = r.sub(r'(\g<0>)',dep)
264
265 if file not in i:
266 i[file] = []
267 i[file].append(dep)
268
269 return provides, requires
270
271 output = subprocess.check_output(shlex.split(rpmdeps) + pkgfiles, stderr=subprocess.STDOUT).decode("utf-8")
272 provides, requires = process_deps(output, pkg, pkgdest, provides, requires)
273
274 return (pkg, provides, requires)
275
276
277def read_shlib_providers(d):
278 import re
279
280 shlib_provider = {}
281 shlibs_dirs = d.getVar('SHLIBSDIRS').split()
282 list_re = re.compile(r'^(.*)\.list$')
283 # Go from least to most specific since the last one found wins
284 for dir in reversed(shlibs_dirs):
285 bb.debug(2, "Reading shlib providers in %s" % (dir))
286 if not os.path.exists(dir):
287 continue
288 for file in sorted(os.listdir(dir)):
289 m = list_re.match(file)
290 if m:
291 dep_pkg = m.group(1)
292 try:
293 fd = open(os.path.join(dir, file))
294 except IOError:
295 # During a build unrelated shlib files may be deleted, so
296 # handle files disappearing between the listdirs and open.
297 continue
298 lines = fd.readlines()
299 fd.close()
300 for l in lines:
301 s = l.strip().split(":")
302 if s[0] not in shlib_provider:
303 shlib_provider[s[0]] = {}
304 shlib_provider[s[0]][s[1]] = (dep_pkg, s[2])
305 return shlib_provider
306
307# We generate a master list of directories to process, we start by
308# seeding this list with reasonable defaults, then load from
309# the fs-perms.txt files
310def fixup_perms(d):
311 import pwd, grp
312
313 cpath = oe.cachedpath.CachedPath()
314 dvar = d.getVar('PKGD')
315
316 # init using a string with the same format as a line as documented in
317 # the fs-perms.txt file
318 # <path> <mode> <uid> <gid> <walk> <fmode> <fuid> <fgid>
319 # <path> link <link target>
320 #
321 # __str__ can be used to print out an entry in the input format
322 #
323 # if fs_perms_entry.path is None:
324 # an error occurred
325 # if fs_perms_entry.link, you can retrieve:
326 # fs_perms_entry.path = path
327 # fs_perms_entry.link = target of link
328 # if not fs_perms_entry.link, you can retrieve:
329 # fs_perms_entry.path = path
330 # fs_perms_entry.mode = expected dir mode or None
331 # fs_perms_entry.uid = expected uid or -1
332 # fs_perms_entry.gid = expected gid or -1
333 # fs_perms_entry.walk = 'true' or something else
334 # fs_perms_entry.fmode = expected file mode or None
335 # fs_perms_entry.fuid = expected file uid or -1
336 # fs_perms_entry_fgid = expected file gid or -1
337 class fs_perms_entry():
338 def __init__(self, line):
339 lsplit = line.split()
340 if len(lsplit) == 3 and lsplit[1].lower() == "link":
341 self._setlink(lsplit[0], lsplit[2])
342 elif len(lsplit) == 8:
343 self._setdir(lsplit[0], lsplit[1], lsplit[2], lsplit[3], lsplit[4], lsplit[5], lsplit[6], lsplit[7])
344 else:
345 msg = "Fixup Perms: invalid config line %s" % line
346 oe.qa.handle_error("perm-config", msg, d)
347 self.path = None
348 self.link = None
349
350 def _setdir(self, path, mode, uid, gid, walk, fmode, fuid, fgid):
351 self.path = os.path.normpath(path)
352 self.link = None
353 self.mode = self._procmode(mode)
354 self.uid = self._procuid(uid)
355 self.gid = self._procgid(gid)
356 self.walk = walk.lower()
357 self.fmode = self._procmode(fmode)
358 self.fuid = self._procuid(fuid)
359 self.fgid = self._procgid(fgid)
360
361 def _setlink(self, path, link):
362 self.path = os.path.normpath(path)
363 self.link = link
364
365 def _procmode(self, mode):
366 if not mode or (mode and mode == "-"):
367 return None
368 else:
369 return int(mode,8)
370
371 # Note uid/gid -1 has special significance in os.lchown
372 def _procuid(self, uid):
373 if uid is None or uid == "-":
374 return -1
375 elif uid.isdigit():
376 return int(uid)
377 else:
378 return pwd.getpwnam(uid).pw_uid
379
380 def _procgid(self, gid):
381 if gid is None or gid == "-":
382 return -1
383 elif gid.isdigit():
384 return int(gid)
385 else:
386 return grp.getgrnam(gid).gr_gid
387
388 # Use for debugging the entries
389 def __str__(self):
390 if self.link:
391 return "%s link %s" % (self.path, self.link)
392 else:
393 mode = "-"
394 if self.mode:
395 mode = "0%o" % self.mode
396 fmode = "-"
397 if self.fmode:
398 fmode = "0%o" % self.fmode
399 uid = self._mapugid(self.uid)
400 gid = self._mapugid(self.gid)
401 fuid = self._mapugid(self.fuid)
402 fgid = self._mapugid(self.fgid)
403 return "%s %s %s %s %s %s %s %s" % (self.path, mode, uid, gid, self.walk, fmode, fuid, fgid)
404
405 def _mapugid(self, id):
406 if id is None or id == -1:
407 return "-"
408 else:
409 return "%d" % id
410
411 # Fix the permission, owner and group of path
412 def fix_perms(path, mode, uid, gid, dir):
413 if mode and not os.path.islink(path):
414 #bb.note("Fixup Perms: chmod 0%o %s" % (mode, dir))
415 os.chmod(path, mode)
416 # -1 is a special value that means don't change the uid/gid
417 # if they are BOTH -1, don't bother to lchown
418 if not (uid == -1 and gid == -1):
419 #bb.note("Fixup Perms: lchown %d:%d %s" % (uid, gid, dir))
420 os.lchown(path, uid, gid)
421
422 # Return a list of configuration files based on either the default
423 # files/fs-perms.txt or the contents of FILESYSTEM_PERMS_TABLES
424 # paths are resolved via BBPATH
425 def get_fs_perms_list(d):
426 str = ""
427 bbpath = d.getVar('BBPATH')
428 fs_perms_tables = d.getVar('FILESYSTEM_PERMS_TABLES') or ""
429 for conf_file in fs_perms_tables.split():
430 confpath = bb.utils.which(bbpath, conf_file)
431 if confpath:
432 str += " %s" % bb.utils.which(bbpath, conf_file)
433 else:
434 bb.warn("cannot find %s specified in FILESYSTEM_PERMS_TABLES" % conf_file)
435 return str
436
437 fs_perms_table = {}
438 fs_link_table = {}
439
440 # By default all of the standard directories specified in
441 # bitbake.conf will get 0755 root:root.
442 target_path_vars = [ 'base_prefix',
443 'prefix',
444 'exec_prefix',
445 'base_bindir',
446 'base_sbindir',
447 'base_libdir',
448 'datadir',
449 'sysconfdir',
450 'servicedir',
451 'sharedstatedir',
452 'localstatedir',
453 'infodir',
454 'mandir',
455 'docdir',
456 'bindir',
457 'sbindir',
458 'libexecdir',
459 'libdir',
460 'includedir' ]
461
462 for path in target_path_vars:
463 dir = d.getVar(path) or ""
464 if dir == "":
465 continue
466 fs_perms_table[dir] = fs_perms_entry(d.expand("%s 0755 root root false - - -" % (dir)))
467
468 # Now we actually load from the configuration files
469 for conf in get_fs_perms_list(d).split():
470 if not os.path.exists(conf):
471 continue
472 with open(conf) as f:
473 for line in f:
474 if line.startswith('#'):
475 continue
476 lsplit = line.split()
477 if len(lsplit) == 0:
478 continue
479 if len(lsplit) != 8 and not (len(lsplit) == 3 and lsplit[1].lower() == "link"):
480 msg = "Fixup perms: %s invalid line: %s" % (conf, line)
481 oe.qa.handle_error("perm-line", msg, d)
482 continue
483 entry = fs_perms_entry(d.expand(line))
484 if entry and entry.path:
485 if entry.link:
486 fs_link_table[entry.path] = entry
487 if entry.path in fs_perms_table:
488 fs_perms_table.pop(entry.path)
489 else:
490 fs_perms_table[entry.path] = entry
491 if entry.path in fs_link_table:
492 fs_link_table.pop(entry.path)
493
494 # Debug -- list out in-memory table
495 #for dir in fs_perms_table:
496 # bb.note("Fixup Perms: %s: %s" % (dir, str(fs_perms_table[dir])))
497 #for link in fs_link_table:
498 # bb.note("Fixup Perms: %s: %s" % (link, str(fs_link_table[link])))
499
500 # We process links first, so we can go back and fixup directory ownership
501 # for any newly created directories
502 # Process in sorted order so /run gets created before /run/lock, etc.
503 for entry in sorted(fs_link_table.values(), key=lambda x: x.link):
504 link = entry.link
505 dir = entry.path
506 origin = dvar + dir
507 if not (cpath.exists(origin) and cpath.isdir(origin) and not cpath.islink(origin)):
508 continue
509
510 if link[0] == "/":
511 target = dvar + link
512 ptarget = link
513 else:
514 target = os.path.join(os.path.dirname(origin), link)
515 ptarget = os.path.join(os.path.dirname(dir), link)
516 if os.path.exists(target):
517 msg = "Fixup Perms: Unable to correct directory link, target already exists: %s -> %s" % (dir, ptarget)
518 oe.qa.handle_error("perm-link", msg, d)
519 continue
520
521 # Create path to move directory to, move it, and then setup the symlink
522 bb.utils.mkdirhier(os.path.dirname(target))
523 #bb.note("Fixup Perms: Rename %s -> %s" % (dir, ptarget))
524 bb.utils.rename(origin, target)
525 #bb.note("Fixup Perms: Link %s -> %s" % (dir, link))
526 os.symlink(link, origin)
527
528 for dir in fs_perms_table:
529 origin = dvar + dir
530 if not (cpath.exists(origin) and cpath.isdir(origin)):
531 continue
532
533 fix_perms(origin, fs_perms_table[dir].mode, fs_perms_table[dir].uid, fs_perms_table[dir].gid, dir)
534
535 if fs_perms_table[dir].walk == 'true':
536 for root, dirs, files in os.walk(origin):
537 for dr in dirs:
538 each_dir = os.path.join(root, dr)
539 fix_perms(each_dir, fs_perms_table[dir].mode, fs_perms_table[dir].uid, fs_perms_table[dir].gid, dir)
540 for f in files:
541 each_file = os.path.join(root, f)
542 fix_perms(each_file, fs_perms_table[dir].fmode, fs_perms_table[dir].fuid, fs_perms_table[dir].fgid, dir)
543
544# Get a list of files from file vars by searching files under current working directory
545# The list contains symlinks, directories and normal files.
546def files_from_filevars(filevars):
547 cpath = oe.cachedpath.CachedPath()
548 files = []
549 for f in filevars:
550 if os.path.isabs(f):
551 f = '.' + f
552 if not f.startswith("./"):
553 f = './' + f
554 globbed = glob.glob(f, recursive=True)
555 if globbed:
556 if [ f ] != globbed:
557 files += globbed
558 continue
559 files.append(f)
560
561 symlink_paths = []
562 for ind, f in enumerate(files):
563 # Handle directory symlinks. Truncate path to the lowest level symlink
564 parent = ''
565 for dirname in f.split('/')[:-1]:
566 parent = os.path.join(parent, dirname)
567 if dirname == '.':
568 continue
569 if cpath.islink(parent):
570 bb.warn("FILES contains file '%s' which resides under a "
571 "directory symlink. Please fix the recipe and use the "
572 "real path for the file." % f[1:])
573 symlink_paths.append(f)
574 files[ind] = parent
575 f = parent
576 break
577
578 if not cpath.islink(f):
579 if cpath.isdir(f):
580 newfiles = [ os.path.join(f,x) for x in os.listdir(f) ]
581 if newfiles:
582 files += newfiles
583
584 return files, symlink_paths
585
586# Called in package_<rpm,ipk,deb>.bbclass to get the correct list of configuration files
587def get_conffiles(pkg, d):
588 pkgdest = d.getVar('PKGDEST')
589 root = os.path.join(pkgdest, pkg)
590 cwd = os.getcwd()
591 os.chdir(root)
592
593 conffiles = d.getVar('CONFFILES:%s' % pkg);
594 if conffiles == None:
595 conffiles = d.getVar('CONFFILES')
596 if conffiles == None:
597 conffiles = ""
598 conffiles = conffiles.split()
599 conf_orig_list = files_from_filevars(conffiles)[0]
600
601 # Remove links and directories from conf_orig_list to get conf_list which only contains normal files
602 conf_list = []
603 for f in conf_orig_list:
604 if os.path.isdir(f):
605 continue
606 if os.path.islink(f):
607 continue
608 if not os.path.exists(f):
609 continue
610 conf_list.append(f)
611
612 # Remove the leading './'
613 for i in range(0, len(conf_list)):
614 conf_list[i] = conf_list[i][1:]
615
616 os.chdir(cwd)
617 return sorted(conf_list)
618
619def legitimize_package_name(s):
620 """
621 Make sure package names are legitimate strings
622 """
623
624 def fixutf(m):
625 cp = m.group(1)
626 if cp:
627 return ('\\u%s' % cp).encode('latin-1').decode('unicode_escape')
628
629 # Handle unicode codepoints encoded as <U0123>, as in glibc locale files.
630 s = re.sub(r'<U([0-9A-Fa-f]{1,4})>', fixutf, s)
631
632 # Remaining package name validity fixes
633 return s.lower().replace('_', '-').replace('@', '+').replace(',', '+').replace('/', '-')
634
635def split_locales(d):
636 cpath = oe.cachedpath.CachedPath()
637 if (d.getVar('PACKAGE_NO_LOCALE') == '1'):
638 bb.debug(1, "package requested not splitting locales")
639 return
640
641 packages = (d.getVar('PACKAGES') or "").split()
642
643 dvar = d.getVar('PKGD')
644 pn = d.getVar('LOCALEBASEPN')
645
646 try:
647 locale_index = packages.index(pn + '-locale')
648 packages.pop(locale_index)
649 except ValueError:
650 locale_index = len(packages)
651
652 localepaths = []
653 locales = set()
654 for localepath in (d.getVar('LOCALE_PATHS') or "").split():
655 localedir = dvar + localepath
656 if not cpath.isdir(localedir):
657 bb.debug(1, 'No locale files in %s' % localepath)
658 continue
659
660 localepaths.append(localepath)
661 with os.scandir(localedir) as it:
662 for entry in it:
663 if entry.is_dir():
664 locales.add(entry.name)
665
666 if len(locales) == 0:
667 bb.debug(1, "No locale files in this package")
668 return
669
670 summary = d.getVar('SUMMARY') or pn
671 description = d.getVar('DESCRIPTION') or ""
672 locale_section = d.getVar('LOCALE_SECTION')
673 mlprefix = d.getVar('MLPREFIX') or ""
674 for l in sorted(locales):
675 ln = legitimize_package_name(l)
676 pkg = pn + '-locale-' + ln
677 packages.insert(locale_index, pkg)
678 locale_index += 1
679 files = []
680 for localepath in localepaths:
681 files.append(os.path.join(localepath, l))
682 d.setVar('FILES:' + pkg, " ".join(files))
683 d.setVar('RRECOMMENDS:' + pkg, '%svirtual-locale-%s' % (mlprefix, ln))
684 d.setVar('RPROVIDES:' + pkg, '%s-locale %s%s-translation' % (pn, mlprefix, ln))
685 d.setVar('SUMMARY:' + pkg, '%s - %s translations' % (summary, l))
686 d.setVar('DESCRIPTION:' + pkg, '%s This package contains language translation files for the %s locale.' % (description, l))
687 if locale_section:
688 d.setVar('SECTION:' + pkg, locale_section)
689
690 d.setVar('PACKAGES', ' '.join(packages))
691
692 # Disabled by RP 18/06/07
693 # Wildcards aren't supported in debian
694 # They break with ipkg since glibc-locale* will mean that
695 # glibc-localedata-translit* won't install as a dependency
696 # for some other package which breaks meta-toolchain
697 # Probably breaks since virtual-locale- isn't provided anywhere
698 #rdep = (d.getVar('RDEPENDS:%s' % pn) or "").split()
699 #rdep.append('%s-locale*' % pn)
700 #d.setVar('RDEPENDS:%s' % pn, ' '.join(rdep))
701
702def package_debug_vars(d):
703 # We default to '.debug' style
704 if d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-file-directory':
705 # Single debug-file-directory style debug info
706 debug_vars = {
707 "append": ".debug",
708 "staticappend": "",
709 "dir": "",
710 "staticdir": "",
711 "libdir": "/usr/lib/debug",
712 "staticlibdir": "/usr/lib/debug-static",
713 "srcdir": "/usr/src/debug",
714 }
715 elif d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-without-src':
716 # Original OE-core, a.k.a. ".debug", style debug info, but without sources in /usr/src/debug
717 debug_vars = {
718 "append": "",
719 "staticappend": "",
720 "dir": "/.debug",
721 "staticdir": "/.debug-static",
722 "libdir": "",
723 "staticlibdir": "",
724 "srcdir": "",
725 }
726 elif d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg':
727 debug_vars = {
728 "append": "",
729 "staticappend": "",
730 "dir": "/.debug",
731 "staticdir": "/.debug-static",
732 "libdir": "",
733 "staticlibdir": "",
734 "srcdir": "/usr/src/debug",
735 }
736 else:
737 # Original OE-core, a.k.a. ".debug", style debug info
738 debug_vars = {
739 "append": "",
740 "staticappend": "",
741 "dir": "/.debug",
742 "staticdir": "/.debug-static",
743 "libdir": "",
744 "staticlibdir": "",
745 "srcdir": "/usr/src/debug",
746 }
747
748 return debug_vars
749
750
751def parse_debugsources_from_dwarfsrcfiles_output(dwarfsrcfiles_output):
752 debugfiles = {}
753
754 for line in dwarfsrcfiles_output.splitlines():
755 if line.startswith("\t"):
756 debugfiles[os.path.normpath(line.split()[0])] = ""
757
758 return debugfiles.keys()
759
760def source_info(file, d, fatal=True):
761 cmd = ["dwarfsrcfiles", file]
762 try:
763 output = subprocess.check_output(cmd, universal_newlines=True, stderr=subprocess.STDOUT)
764 retval = 0
765 except subprocess.CalledProcessError as exc:
766 output = exc.output
767 retval = exc.returncode
768
769 # 255 means a specific file wasn't fully parsed to get the debug file list, which is not a fatal failure
770 if retval != 0 and retval != 255:
771 msg = "dwarfsrcfiles failed with exit code %s (cmd was %s)%s" % (retval, cmd, ":\n%s" % output if output else "")
772 if fatal:
773 bb.fatal(msg)
774 bb.note(msg)
775
776 debugsources = parse_debugsources_from_dwarfsrcfiles_output(output)
777
778 return list(debugsources)
779
780def splitdebuginfo(file, dvar, dv, d):
781 # Function to split a single file into two components, one is the stripped
782 # target system binary, the other contains any debugging information. The
783 # two files are linked to reference each other.
784 #
785 # return a mapping of files:debugsources
786
787 src = file[len(dvar):]
788 dest = dv["libdir"] + os.path.dirname(src) + dv["dir"] + "/" + os.path.basename(src) + dv["append"]
789 debugfile = dvar + dest
790 sources = []
791
792 if file.endswith(".ko") and file.find("/lib/modules/") != -1:
793 if oe.package.is_kernel_module_signed(file):
794 bb.debug(1, "Skip strip on signed module %s" % file)
795 return (file, sources)
796
797 # Split the file...
798 bb.utils.mkdirhier(os.path.dirname(debugfile))
799 #bb.note("Split %s -> %s" % (file, debugfile))
800 # Only store off the hard link reference if we successfully split!
801
802 dvar = d.getVar('PKGD')
803 objcopy = d.getVar("OBJCOPY")
804
805 newmode = None
806 if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
807 origmode = os.stat(file)[stat.ST_MODE]
808 newmode = origmode | stat.S_IWRITE | stat.S_IREAD
809 os.chmod(file, newmode)
810
811 # We need to extract the debug src information here...
812 if dv["srcdir"]:
813 sources = source_info(file, d)
814
815 bb.utils.mkdirhier(os.path.dirname(debugfile))
816
817 subprocess.check_output([objcopy, '--only-keep-debug', file, debugfile], stderr=subprocess.STDOUT)
818
819 # Set the debuglink to have the view of the file path on the target
820 subprocess.check_output([objcopy, '--add-gnu-debuglink', debugfile, file], stderr=subprocess.STDOUT)
821
822 if newmode:
823 os.chmod(file, origmode)
824
825 return (file, sources)
826
827def splitstaticdebuginfo(file, dvar, dv, d):
828 # Unlike the function above, there is no way to split a static library
829 # two components. So to get similar results we will copy the unmodified
830 # static library (containing the debug symbols) into a new directory.
831 # We will then strip (preserving symbols) the static library in the
832 # typical location.
833 #
834 # return a mapping of files:debugsources
835
836 src = file[len(dvar):]
837 dest = dv["staticlibdir"] + os.path.dirname(src) + dv["staticdir"] + "/" + os.path.basename(src) + dv["staticappend"]
838 debugfile = dvar + dest
839 sources = []
840
841 # Copy the file...
842 bb.utils.mkdirhier(os.path.dirname(debugfile))
843 #bb.note("Copy %s -> %s" % (file, debugfile))
844
845 dvar = d.getVar('PKGD')
846
847 newmode = None
848 if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
849 origmode = os.stat(file)[stat.ST_MODE]
850 newmode = origmode | stat.S_IWRITE | stat.S_IREAD
851 os.chmod(file, newmode)
852
853 # We need to extract the debug src information here...
854 if dv["srcdir"]:
855 sources = source_info(file, d)
856
857 bb.utils.mkdirhier(os.path.dirname(debugfile))
858
859 # Copy the unmodified item to the debug directory
860 shutil.copy2(file, debugfile)
861
862 if newmode:
863 os.chmod(file, origmode)
864
865 return (file, sources)
866
867def inject_minidebuginfo(file, dvar, dv, d):
868 # Extract just the symbols from debuginfo into minidebuginfo,
869 # compress it with xz and inject it back into the binary in a .gnu_debugdata section.
870 # https://sourceware.org/gdb/onlinedocs/gdb/MiniDebugInfo.html
871
872 readelf = d.getVar('READELF')
873 nm = d.getVar('NM')
874 objcopy = d.getVar('OBJCOPY')
875
876 minidebuginfodir = d.expand('${WORKDIR}/minidebuginfo')
877
878 src = file[len(dvar):]
879 dest = dv["libdir"] + os.path.dirname(src) + dv["dir"] + "/" + os.path.basename(src) + dv["append"]
880 debugfile = dvar + dest
881 minidebugfile = minidebuginfodir + src + '.minidebug'
882 bb.utils.mkdirhier(os.path.dirname(minidebugfile))
883
884 # If we didn't produce debuginfo for any reason, we can't produce minidebuginfo either
885 # so skip it.
886 if not os.path.exists(debugfile):
887 bb.debug(1, 'ELF file {} has no debuginfo, skipping minidebuginfo injection'.format(file))
888 return
889
890 # minidebuginfo does not make sense to apply to ELF objects other than
891 # executables and shared libraries, skip applying the minidebuginfo
892 # generation for objects like kernel modules.
893 for line in subprocess.check_output([readelf, '-h', debugfile], universal_newlines=True).splitlines():
894 if not line.strip().startswith("Type:"):
895 continue
896 elftype = line.split(":")[1].strip()
897 if not any(elftype.startswith(i) for i in ["EXEC", "DYN"]):
898 bb.debug(1, 'ELF file {} is not executable/shared, skipping minidebuginfo injection'.format(file))
899 return
900 break
901
902 # Find non-allocated PROGBITS, NOTE, and NOBITS sections in the debuginfo.
903 # We will exclude all of these from minidebuginfo to save space.
904 remove_section_names = []
905 for line in subprocess.check_output([readelf, '-W', '-S', debugfile], universal_newlines=True).splitlines():
906 # strip the leading " [ 1]" section index to allow splitting on space
907 if ']' not in line:
908 continue
909 fields = line[line.index(']') + 1:].split()
910 if len(fields) < 7:
911 continue
912 name = fields[0]
913 type = fields[1]
914 flags = fields[6]
915 # .debug_ sections will be removed by objcopy -S so no need to explicitly remove them
916 if name.startswith('.debug_'):
917 continue
918 if 'A' not in flags and type in ['PROGBITS', 'NOTE', 'NOBITS']:
919 remove_section_names.append(name)
920
921 # List dynamic symbols in the binary. We can exclude these from minidebuginfo
922 # because they are always present in the binary.
923 dynsyms = set()
924 for line in subprocess.check_output([nm, '-D', file, '--format=posix', '--defined-only'], universal_newlines=True).splitlines():
925 dynsyms.add(line.split()[0])
926
927 # Find all function symbols from debuginfo which aren't in the dynamic symbols table.
928 # These are the ones we want to keep in minidebuginfo.
929 keep_symbols_file = minidebugfile + '.symlist'
930 found_any_symbols = False
931 with open(keep_symbols_file, 'w') as f:
932 for line in subprocess.check_output([nm, debugfile, '--format=sysv', '--defined-only'], universal_newlines=True).splitlines():
933 fields = line.split('|')
934 if len(fields) < 7:
935 continue
936 name = fields[0].strip()
937 type = fields[3].strip()
938 if type == 'FUNC' and name not in dynsyms:
939 f.write('{}\n'.format(name))
940 found_any_symbols = True
941
942 if not found_any_symbols:
943 bb.debug(1, 'ELF file {} contains no symbols, skipping minidebuginfo injection'.format(file))
944 return
945
946 bb.utils.remove(minidebugfile)
947 bb.utils.remove(minidebugfile + '.xz')
948
949 subprocess.check_call([objcopy, '-S'] +
950 ['--remove-section={}'.format(s) for s in remove_section_names] +
951 ['--keep-symbols={}'.format(keep_symbols_file), debugfile, minidebugfile])
952
953 subprocess.check_call(['xz', '--keep', minidebugfile])
954
955 subprocess.check_call([objcopy, '--add-section', '.gnu_debugdata={}.xz'.format(minidebugfile), file])
956
957def copydebugsources(debugsrcdir, sources, d):
958 # The debug src information written out to sourcefile is further processed
959 # and copied to the destination here.
960
961 cpath = oe.cachedpath.CachedPath()
962
963 if debugsrcdir and sources:
964 sourcefile = d.expand("${WORKDIR}/debugsources.list")
965 bb.utils.remove(sourcefile)
966
967 # filenames are null-separated - this is an artefact of the previous use
968 # of rpm's debugedit, which was writing them out that way, and the code elsewhere
969 # is still assuming that.
970 debuglistoutput = '\0'.join(sources) + '\0'
971 with open(sourcefile, 'a') as sf:
972 sf.write(debuglistoutput)
973
974 dvar = d.getVar('PKGD')
975 strip = d.getVar("STRIP")
976 objcopy = d.getVar("OBJCOPY")
977 workdir = d.getVar("WORKDIR")
978 sdir = d.getVar("S")
979 cflags = d.expand("${CFLAGS}")
980
981 prefixmap = {}
982 for flag in cflags.split():
983 if not flag.startswith("-fdebug-prefix-map"):
984 continue
985 if "recipe-sysroot" in flag:
986 continue
987 flag = flag.split("=")
988 prefixmap[flag[1]] = flag[2]
989
990 nosuchdir = []
991 basepath = dvar
992 for p in debugsrcdir.split("/"):
993 basepath = basepath + "/" + p
994 if not cpath.exists(basepath):
995 nosuchdir.append(basepath)
996 bb.utils.mkdirhier(basepath)
997 cpath.updatecache(basepath)
998
999 for pmap in prefixmap:
1000 # Ignore files from the recipe sysroots (target and native)
1001 cmd = "LC_ALL=C ; sort -z -u '%s' | egrep -v -z '((<internal>|<built-in>)$|/.*recipe-sysroot.*/)' | " % sourcefile
1002 # We need to ignore files that are not actually ours
1003 # we do this by only paying attention to items from this package
1004 cmd += "fgrep -zw '%s' | " % prefixmap[pmap]
1005 # Remove prefix in the source paths
1006 cmd += "sed 's#%s/##g' | " % (prefixmap[pmap])
1007 cmd += "(cd '%s' ; cpio -pd0mlL --no-preserve-owner '%s%s' 2>/dev/null)" % (pmap, dvar, prefixmap[pmap])
1008
1009 try:
1010 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
1011 except subprocess.CalledProcessError:
1012 # Can "fail" if internal headers/transient sources are attempted
1013 pass
1014 # cpio seems to have a bug with -lL together and symbolic links are just copied, not dereferenced.
1015 # Work around this by manually finding and copying any symbolic links that made it through.
1016 cmd = "find %s%s -type l -print0 -delete | sed s#%s%s/##g | (cd '%s' ; cpio -pd0mL --no-preserve-owner '%s%s')" % \
1017 (dvar, prefixmap[pmap], dvar, prefixmap[pmap], pmap, dvar, prefixmap[pmap])
1018 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
1019
1020 # debugsources.list may be polluted from the host if we used externalsrc,
1021 # cpio uses copy-pass and may have just created a directory structure
1022 # matching the one from the host, if thats the case move those files to
1023 # debugsrcdir to avoid host contamination.
1024 # Empty dir structure will be deleted in the next step.
1025
1026 # Same check as above for externalsrc
1027 if workdir not in sdir:
1028 if os.path.exists(dvar + debugsrcdir + sdir):
1029 cmd = "mv %s%s%s/* %s%s" % (dvar, debugsrcdir, sdir, dvar,debugsrcdir)
1030 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
1031
1032 # The copy by cpio may have resulted in some empty directories! Remove these
1033 cmd = "find %s%s -empty -type d -delete" % (dvar, debugsrcdir)
1034 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
1035
1036 # Also remove debugsrcdir if its empty
1037 for p in nosuchdir[::-1]:
1038 if os.path.exists(p) and not os.listdir(p):
1039 os.rmdir(p)
1040
1041
1042def process_split_and_strip_files(d):
1043 cpath = oe.cachedpath.CachedPath()
1044
1045 dvar = d.getVar('PKGD')
1046 pn = d.getVar('PN')
1047 hostos = d.getVar('HOST_OS')
1048
1049 oldcwd = os.getcwd()
1050 os.chdir(dvar)
1051
1052 dv = package_debug_vars(d)
1053
1054 #
1055 # First lets figure out all of the files we may have to process ... do this only once!
1056 #
1057 elffiles = {}
1058 symlinks = {}
1059 staticlibs = []
1060 inodes = {}
1061 libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir"))
1062 baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir"))
1063 skipfiles = (d.getVar("INHIBIT_PACKAGE_STRIP_FILES") or "").split()
1064 if (d.getVar('INHIBIT_PACKAGE_STRIP') != '1' or \
1065 d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'):
1066 checkelf = {}
1067 checkelflinks = {}
1068 checkstatic = {}
1069 for root, dirs, files in cpath.walk(dvar):
1070 for f in files:
1071 file = os.path.join(root, f)
1072
1073 # Skip debug files
1074 if dv["append"] and file.endswith(dv["append"]):
1075 continue
1076 if dv["dir"] and dv["dir"] in os.path.dirname(file[len(dvar):]):
1077 continue
1078
1079 if file in skipfiles:
1080 continue
1081
1082 try:
1083 ltarget = cpath.realpath(file, dvar, False)
1084 s = cpath.lstat(ltarget)
1085 except OSError as e:
1086 (err, strerror) = e.args
1087 if err != errno.ENOENT:
1088 raise
1089 # Skip broken symlinks
1090 continue
1091 if not s:
1092 continue
1093
1094 if oe.package.is_static_lib(file):
1095 # Use a reference of device ID and inode number to identify files
1096 file_reference = "%d_%d" % (s.st_dev, s.st_ino)
1097 checkstatic[file] = (file, file_reference)
1098 continue
1099
1100 # Check its an executable
1101 if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) \
1102 or (s[stat.ST_MODE] & stat.S_IXOTH) \
1103 or ((file.startswith(libdir) or file.startswith(baselibdir)) \
1104 and (".so" in f or ".node" in f)) \
1105 or (f.startswith('vmlinux') or ".ko" in f):
1106
1107 if cpath.islink(file):
1108 checkelflinks[file] = ltarget
1109 continue
1110 # Use a reference of device ID and inode number to identify files
1111 file_reference = "%d_%d" % (s.st_dev, s.st_ino)
1112 checkelf[file] = (file, file_reference)
1113
1114 results = oe.utils.multiprocess_launch(oe.package.is_elf, checkelflinks.values(), d)
1115 results_map = {}
1116 for (ltarget, elf_file) in results:
1117 results_map[ltarget] = elf_file
1118 for file in checkelflinks:
1119 ltarget = checkelflinks[file]
1120 # If it's a symlink, and points to an ELF file, we capture the readlink target
1121 if results_map[ltarget]:
1122 target = os.readlink(file)
1123 #bb.note("Sym: %s (%d)" % (ltarget, results_map[ltarget]))
1124 symlinks[file] = target
1125
1126 results = oe.utils.multiprocess_launch(oe.package.is_elf, checkelf.keys(), d)
1127
1128 # Sort results by file path. This ensures that the files are always
1129 # processed in the same order, which is important to make sure builds
1130 # are reproducible when dealing with hardlinks
1131 results.sort(key=lambda x: x[0])
1132
1133 for (file, elf_file) in results:
1134 # It's a file (or hardlink), not a link
1135 # ...but is it ELF, and is it already stripped?
1136 if elf_file & 1:
1137 if elf_file & 2:
1138 if 'already-stripped' in (d.getVar('INSANE_SKIP:' + pn) or "").split():
1139 bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dvar):], pn))
1140 else:
1141 msg = "File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn)
1142 oe.qa.handle_error("already-stripped", msg, d)
1143 continue
1144
1145 # At this point we have an unstripped elf file. We need to:
1146 # a) Make sure any file we strip is not hardlinked to anything else outside this tree
1147 # b) Only strip any hardlinked file once (no races)
1148 # c) Track any hardlinks between files so that we can reconstruct matching debug file hardlinks
1149
1150 # Use a reference of device ID and inode number to identify files
1151 file_reference = checkelf[file][1]
1152 if file_reference in inodes:
1153 os.unlink(file)
1154 os.link(inodes[file_reference][0], file)
1155 inodes[file_reference].append(file)
1156 else:
1157 inodes[file_reference] = [file]
1158 # break hardlink
1159 bb.utils.break_hardlinks(file)
1160 elffiles[file] = elf_file
1161 # Modified the file so clear the cache
1162 cpath.updatecache(file)
1163
1164 # Do the same hardlink processing as above, but for static libraries
1165 results = list(checkstatic.keys())
1166
1167 # As above, sort the results.
1168 results.sort(key=lambda x: x[0])
1169
1170 for file in results:
1171 # Use a reference of device ID and inode number to identify files
1172 file_reference = checkstatic[file][1]
1173 if file_reference in inodes:
1174 os.unlink(file)
1175 os.link(inodes[file_reference][0], file)
1176 inodes[file_reference].append(file)
1177 else:
1178 inodes[file_reference] = [file]
1179 # break hardlink
1180 bb.utils.break_hardlinks(file)
1181 staticlibs.append(file)
1182 # Modified the file so clear the cache
1183 cpath.updatecache(file)
1184
1185 def strip_pkgd_prefix(f):
1186 nonlocal dvar
1187
1188 if f.startswith(dvar):
1189 return f[len(dvar):]
1190
1191 return f
1192
1193 #
1194 # First lets process debug splitting
1195 #
1196 if (d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'):
1197 results = oe.utils.multiprocess_launch(splitdebuginfo, list(elffiles), d, extraargs=(dvar, dv, d))
1198
1199 if dv["srcdir"] and not hostos.startswith("mingw"):
1200 if (d.getVar('PACKAGE_DEBUG_STATIC_SPLIT') == '1'):
1201 results = oe.utils.multiprocess_launch(splitstaticdebuginfo, staticlibs, d, extraargs=(dvar, dv, d))
1202 else:
1203 for file in staticlibs:
1204 results.append( (file,source_info(file, d)) )
1205
1206 d.setVar("PKGDEBUGSOURCES", {strip_pkgd_prefix(f): sorted(s) for f, s in results})
1207
1208 sources = set()
1209 for r in results:
1210 sources.update(r[1])
1211
1212 # Hardlink our debug symbols to the other hardlink copies
1213 for ref in inodes:
1214 if len(inodes[ref]) == 1:
1215 continue
1216
1217 target = inodes[ref][0][len(dvar):]
1218 for file in inodes[ref][1:]:
1219 src = file[len(dvar):]
1220 dest = dv["libdir"] + os.path.dirname(src) + dv["dir"] + "/" + os.path.basename(target) + dv["append"]
1221 fpath = dvar + dest
1222 ftarget = dvar + dv["libdir"] + os.path.dirname(target) + dv["dir"] + "/" + os.path.basename(target) + dv["append"]
1223 if os.access(ftarget, os.R_OK):
1224 bb.utils.mkdirhier(os.path.dirname(fpath))
1225 # Only one hardlink of separated debug info file in each directory
1226 if not os.access(fpath, os.R_OK):
1227 #bb.note("Link %s -> %s" % (fpath, ftarget))
1228 os.link(ftarget, fpath)
1229 elif (d.getVar('PACKAGE_DEBUG_STATIC_SPLIT') == '1'):
1230 deststatic = dv["staticlibdir"] + os.path.dirname(src) + dv["staticdir"] + "/" + os.path.basename(file) + dv["staticappend"]
1231 fpath = dvar + deststatic
1232 ftarget = dvar + dv["staticlibdir"] + os.path.dirname(target) + dv["staticdir"] + "/" + os.path.basename(target) + dv["staticappend"]
1233 if os.access(ftarget, os.R_OK):
1234 bb.utils.mkdirhier(os.path.dirname(fpath))
1235 # Only one hardlink of separated debug info file in each directory
1236 if not os.access(fpath, os.R_OK):
1237 #bb.note("Link %s -> %s" % (fpath, ftarget))
1238 os.link(ftarget, fpath)
1239 else:
1240 bb.note("Unable to find inode link target %s" % (target))
1241
1242 # Create symlinks for all cases we were able to split symbols
1243 for file in symlinks:
1244 src = file[len(dvar):]
1245 dest = dv["libdir"] + os.path.dirname(src) + dv["dir"] + "/" + os.path.basename(src) + dv["append"]
1246 fpath = dvar + dest
1247 # Skip it if the target doesn't exist
1248 try:
1249 s = os.stat(fpath)
1250 except OSError as e:
1251 (err, strerror) = e.args
1252 if err != errno.ENOENT:
1253 raise
1254 continue
1255
1256 ltarget = symlinks[file]
1257 lpath = os.path.dirname(ltarget)
1258 lbase = os.path.basename(ltarget)
1259 ftarget = ""
1260 if lpath and lpath != ".":
1261 ftarget += lpath + dv["dir"] + "/"
1262 ftarget += lbase + dv["append"]
1263 if lpath.startswith(".."):
1264 ftarget = os.path.join("..", ftarget)
1265 bb.utils.mkdirhier(os.path.dirname(fpath))
1266 #bb.note("Symlink %s -> %s" % (fpath, ftarget))
1267 os.symlink(ftarget, fpath)
1268
1269 # Process the dv["srcdir"] if requested...
1270 # This copies and places the referenced sources for later debugging...
1271 copydebugsources(dv["srcdir"], sources, d)
1272 #
1273 # End of debug splitting
1274 #
1275
1276 #
1277 # Now lets go back over things and strip them
1278 #
1279 if (d.getVar('INHIBIT_PACKAGE_STRIP') != '1'):
1280 strip = d.getVar("STRIP")
1281 sfiles = []
1282 for file in elffiles:
1283 elf_file = int(elffiles[file])
1284 #bb.note("Strip %s" % file)
1285 sfiles.append((file, elf_file, strip))
1286 if (d.getVar('PACKAGE_STRIP_STATIC') == '1' or d.getVar('PACKAGE_DEBUG_STATIC_SPLIT') == '1'):
1287 for f in staticlibs:
1288 sfiles.append((f, 16, strip))
1289
1290 oe.utils.multiprocess_launch(oe.package.runstrip, sfiles, d)
1291
1292 # Build "minidebuginfo" and reinject it back into the stripped binaries
1293 if bb.utils.contains('DISTRO_FEATURES', 'minidebuginfo', True, False, d):
1294 oe.utils.multiprocess_launch(inject_minidebuginfo, list(elffiles), d,
1295 extraargs=(dvar, dv, d))
1296
1297 #
1298 # End of strip
1299 #
1300 os.chdir(oldcwd)
1301
1302
1303def populate_packages(d):
1304 cpath = oe.cachedpath.CachedPath()
1305
1306 workdir = d.getVar('WORKDIR')
1307 outdir = d.getVar('DEPLOY_DIR')
1308 dvar = d.getVar('PKGD')
1309 packages = d.getVar('PACKAGES').split()
1310 pn = d.getVar('PN')
1311
1312 bb.utils.mkdirhier(outdir)
1313 os.chdir(dvar)
1314
1315 autodebug = not (d.getVar("NOAUTOPACKAGEDEBUG") or False)
1316
1317 split_source_package = (d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg')
1318
1319 # If debug-with-srcpkg mode is enabled then add the source package if it
1320 # doesn't exist and add the source file contents to the source package.
1321 if split_source_package:
1322 src_package_name = ('%s-src' % d.getVar('PN'))
1323 if not src_package_name in packages:
1324 packages.append(src_package_name)
1325 d.setVar('FILES:%s' % src_package_name, '/usr/src/debug')
1326
1327 # Sanity check PACKAGES for duplicates
1328 # Sanity should be moved to sanity.bbclass once we have the infrastructure
1329 package_dict = {}
1330
1331 for i, pkg in enumerate(packages):
1332 if pkg in package_dict:
1333 msg = "%s is listed in PACKAGES multiple times, this leads to packaging errors." % pkg
1334 oe.qa.handle_error("packages-list", msg, d)
1335 # Ensure the source package gets the chance to pick up the source files
1336 # before the debug package by ordering it first in PACKAGES. Whether it
1337 # actually picks up any source files is controlled by
1338 # PACKAGE_DEBUG_SPLIT_STYLE.
1339 elif pkg.endswith("-src"):
1340 package_dict[pkg] = (10, i)
1341 elif autodebug and pkg.endswith("-dbg"):
1342 package_dict[pkg] = (30, i)
1343 else:
1344 package_dict[pkg] = (50, i)
1345 packages = sorted(package_dict.keys(), key=package_dict.get)
1346 d.setVar('PACKAGES', ' '.join(packages))
1347 pkgdest = d.getVar('PKGDEST')
1348
1349 seen = []
1350
1351 # os.mkdir masks the permissions with umask so we have to unset it first
1352 oldumask = os.umask(0)
1353
1354 debug = []
1355 for root, dirs, files in cpath.walk(dvar):
1356 dir = root[len(dvar):]
1357 if not dir:
1358 dir = os.sep
1359 for f in (files + dirs):
1360 path = "." + os.path.join(dir, f)
1361 if "/.debug/" in path or "/.debug-static/" in path or path.endswith("/.debug"):
1362 debug.append(path)
1363
1364 for pkg in packages:
1365 root = os.path.join(pkgdest, pkg)
1366 bb.utils.mkdirhier(root)
1367
1368 filesvar = d.getVar('FILES:%s' % pkg) or ""
1369 if "//" in filesvar:
1370 msg = "FILES variable for package %s contains '//' which is invalid. Attempting to fix this but you should correct the metadata.\n" % pkg
1371 oe.qa.handle_error("files-invalid", msg, d)
1372 filesvar.replace("//", "/")
1373
1374 origfiles = filesvar.split()
1375 files, symlink_paths = oe.package.files_from_filevars(origfiles)
1376
1377 if autodebug and pkg.endswith("-dbg"):
1378 files.extend(debug)
1379
1380 for file in files:
1381 if (not cpath.islink(file)) and (not cpath.exists(file)):
1382 continue
1383 if file in seen:
1384 continue
1385 seen.append(file)
1386
1387 def mkdir(src, dest, p):
1388 src = os.path.join(src, p)
1389 dest = os.path.join(dest, p)
1390 fstat = cpath.stat(src)
1391 os.mkdir(dest)
1392 os.chmod(dest, fstat.st_mode)
1393 os.chown(dest, fstat.st_uid, fstat.st_gid)
1394 if p not in seen:
1395 seen.append(p)
1396 cpath.updatecache(dest)
1397
1398 def mkdir_recurse(src, dest, paths):
1399 if cpath.exists(dest + '/' + paths):
1400 return
1401 while paths.startswith("./"):
1402 paths = paths[2:]
1403 p = "."
1404 for c in paths.split("/"):
1405 p = os.path.join(p, c)
1406 if not cpath.exists(os.path.join(dest, p)):
1407 mkdir(src, dest, p)
1408
1409 if cpath.isdir(file) and not cpath.islink(file):
1410 mkdir_recurse(dvar, root, file)
1411 continue
1412
1413 mkdir_recurse(dvar, root, os.path.dirname(file))
1414 fpath = os.path.join(root,file)
1415 if not cpath.islink(file):
1416 os.link(file, fpath)
1417 continue
1418 ret = bb.utils.copyfile(file, fpath)
1419 if ret is False or ret == 0:
1420 bb.fatal("File population failed")
1421
1422 # Check if symlink paths exist
1423 for file in symlink_paths:
1424 if not os.path.exists(os.path.join(root,file)):
1425 bb.fatal("File '%s' cannot be packaged into '%s' because its "
1426 "parent directory structure does not exist. One of "
1427 "its parent directories is a symlink whose target "
1428 "directory is not included in the package." %
1429 (file, pkg))
1430
1431 os.umask(oldumask)
1432 os.chdir(workdir)
1433
1434 # Handle excluding packages with incompatible licenses
1435 package_list = []
1436 for pkg in packages:
1437 licenses = d.getVar('_exclude_incompatible-' + pkg)
1438 if licenses:
1439 msg = "Excluding %s from packaging as it has incompatible license(s): %s" % (pkg, licenses)
1440 oe.qa.handle_error("incompatible-license", msg, d)
1441 else:
1442 package_list.append(pkg)
1443 d.setVar('PACKAGES', ' '.join(package_list))
1444
1445 unshipped = []
1446 for root, dirs, files in cpath.walk(dvar):
1447 dir = root[len(dvar):]
1448 if not dir:
1449 dir = os.sep
1450 for f in (files + dirs):
1451 path = os.path.join(dir, f)
1452 if ('.' + path) not in seen:
1453 unshipped.append(path)
1454
1455 if unshipped != []:
1456 msg = pn + ": Files/directories were installed but not shipped in any package:"
1457 if "installed-vs-shipped" in (d.getVar('INSANE_SKIP:' + pn) or "").split():
1458 bb.note("Package %s skipping QA tests: installed-vs-shipped" % pn)
1459 else:
1460 for f in unshipped:
1461 msg = msg + "\n " + f
1462 msg = msg + "\nPlease set FILES such that these items are packaged. Alternatively if they are unneeded, avoid installing them or delete them within do_install.\n"
1463 msg = msg + "%s: %d installed and not shipped files." % (pn, len(unshipped))
1464 oe.qa.handle_error("installed-vs-shipped", msg, d)
1465
1466def process_fixsymlinks(pkgfiles, d):
1467 cpath = oe.cachedpath.CachedPath()
1468 pkgdest = d.getVar('PKGDEST')
1469 packages = d.getVar("PACKAGES", False).split()
1470
1471 dangling_links = {}
1472 pkg_files = {}
1473 for pkg in packages:
1474 dangling_links[pkg] = []
1475 pkg_files[pkg] = []
1476 inst_root = os.path.join(pkgdest, pkg)
1477 for path in pkgfiles[pkg]:
1478 rpath = path[len(inst_root):]
1479 pkg_files[pkg].append(rpath)
1480 rtarget = cpath.realpath(path, inst_root, True, assume_dir = True)
1481 if not cpath.lexists(rtarget):
1482 dangling_links[pkg].append(os.path.normpath(rtarget[len(inst_root):]))
1483
1484 newrdepends = {}
1485 for pkg in dangling_links:
1486 for l in dangling_links[pkg]:
1487 found = False
1488 bb.debug(1, "%s contains dangling link %s" % (pkg, l))
1489 for p in packages:
1490 if l in pkg_files[p]:
1491 found = True
1492 bb.debug(1, "target found in %s" % p)
1493 if p == pkg:
1494 break
1495 if pkg not in newrdepends:
1496 newrdepends[pkg] = []
1497 newrdepends[pkg].append(p)
1498 break
1499 if found == False:
1500 bb.note("%s contains dangling symlink to %s" % (pkg, l))
1501
1502 for pkg in newrdepends:
1503 rdepends = bb.utils.explode_dep_versions2(d.getVar('RDEPENDS:' + pkg) or "")
1504 for p in newrdepends[pkg]:
1505 if p not in rdepends:
1506 rdepends[p] = []
1507 d.setVar('RDEPENDS:' + pkg, bb.utils.join_deps(rdepends, commasep=False))
1508
1509def process_filedeps(pkgfiles, d):
1510 """
1511 Collect perfile run-time dependency metadata
1512 Output:
1513 FILERPROVIDESFLIST:pkg - list of all files w/ deps
1514 FILERPROVIDES:filepath:pkg - per file dep
1515
1516 FILERDEPENDSFLIST:pkg - list of all files w/ deps
1517 FILERDEPENDS:filepath:pkg - per file dep
1518 """
1519 if d.getVar('SKIP_FILEDEPS') == '1':
1520 return
1521
1522 pkgdest = d.getVar('PKGDEST')
1523 packages = d.getVar('PACKAGES')
1524 rpmdeps = d.getVar('RPMDEPS')
1525
1526 def chunks(files, n):
1527 return [files[i:i+n] for i in range(0, len(files), n)]
1528
1529 pkglist = []
1530 for pkg in packages.split():
1531 if d.getVar('SKIP_FILEDEPS:' + pkg) == '1':
1532 continue
1533 if pkg.endswith('-dbg') or pkg.endswith('-doc') or pkg.find('-locale-') != -1 or pkg.find('-localedata-') != -1 or pkg.find('-gconv-') != -1 or pkg.find('-charmap-') != -1 or pkg.startswith('kernel-module-') or pkg.endswith('-src'):
1534 continue
1535 for files in chunks(pkgfiles[pkg], 100):
1536 pkglist.append((pkg, files, rpmdeps, pkgdest))
1537
1538 processed = oe.utils.multiprocess_launch(oe.package.filedeprunner, pkglist, d)
1539
1540 provides_files = {}
1541 requires_files = {}
1542
1543 for result in processed:
1544 (pkg, provides, requires) = result
1545
1546 if pkg not in provides_files:
1547 provides_files[pkg] = []
1548 if pkg not in requires_files:
1549 requires_files[pkg] = []
1550
1551 for file in sorted(provides):
1552 provides_files[pkg].append(file)
1553 key = "FILERPROVIDES:" + file + ":" + pkg
1554 d.appendVar(key, " " + " ".join(provides[file]))
1555
1556 for file in sorted(requires):
1557 requires_files[pkg].append(file)
1558 key = "FILERDEPENDS:" + file + ":" + pkg
1559 d.appendVar(key, " " + " ".join(requires[file]))
1560
1561 for pkg in requires_files:
1562 d.setVar("FILERDEPENDSFLIST:" + pkg, " ".join(sorted(requires_files[pkg])))
1563 for pkg in provides_files:
1564 d.setVar("FILERPROVIDESFLIST:" + pkg, " ".join(sorted(provides_files[pkg])))
1565
1566def process_shlibs(pkgfiles, d):
1567 cpath = oe.cachedpath.CachedPath()
1568
1569 exclude_shlibs = d.getVar('EXCLUDE_FROM_SHLIBS', False)
1570 if exclude_shlibs:
1571 bb.note("not generating shlibs")
1572 return
1573
1574 lib_re = re.compile(r"^.*\.so")
1575 libdir_re = re.compile(r".*/%s$" % d.getVar('baselib'))
1576
1577 packages = d.getVar('PACKAGES')
1578
1579 shlib_pkgs = []
1580 exclusion_list = d.getVar("EXCLUDE_PACKAGES_FROM_SHLIBS")
1581 if exclusion_list:
1582 for pkg in packages.split():
1583 if pkg not in exclusion_list.split():
1584 shlib_pkgs.append(pkg)
1585 else:
1586 bb.note("not generating shlibs for %s" % pkg)
1587 else:
1588 shlib_pkgs = packages.split()
1589
1590 hostos = d.getVar('HOST_OS')
1591
1592 workdir = d.getVar('WORKDIR')
1593
1594 ver = d.getVar('PKGV')
1595 if not ver:
1596 msg = "PKGV not defined"
1597 oe.qa.handle_error("pkgv-undefined", msg, d)
1598 return
1599
1600 pkgdest = d.getVar('PKGDEST')
1601
1602 shlibswork_dir = d.getVar('SHLIBSWORKDIR')
1603
1604 def linux_so(file, pkg, pkgver, d):
1605 needs_ldconfig = False
1606 needed = set()
1607 sonames = set()
1608 renames = []
1609 ldir = os.path.dirname(file).replace(pkgdest + "/" + pkg, '')
1610 cmd = d.getVar('OBJDUMP') + " -p " + shlex.quote(file) + " 2>/dev/null"
1611 fd = os.popen(cmd)
1612 lines = fd.readlines()
1613 fd.close()
1614 rpath = tuple()
1615 for l in lines:
1616 m = re.match(r"\s+RPATH\s+([^\s]*)", l)
1617 if m:
1618 rpaths = m.group(1).replace("$ORIGIN", ldir).split(":")
1619 rpath = tuple(map(os.path.normpath, rpaths))
1620 for l in lines:
1621 m = re.match(r"\s+NEEDED\s+([^\s]*)", l)
1622 if m:
1623 dep = m.group(1)
1624 if dep not in needed:
1625 needed.add((dep, file, rpath))
1626 m = re.match(r"\s+SONAME\s+([^\s]*)", l)
1627 if m:
1628 this_soname = m.group(1)
1629 prov = (this_soname, ldir, pkgver)
1630 if not prov in sonames:
1631 # if library is private (only used by package) then do not build shlib for it
1632 if not private_libs or len([i for i in private_libs if fnmatch.fnmatch(this_soname, i)]) == 0:
1633 sonames.add(prov)
1634 if libdir_re.match(os.path.dirname(file)):
1635 needs_ldconfig = True
1636 if needs_ldconfig and snap_symlinks and (os.path.basename(file) != this_soname):
1637 renames.append((file, os.path.join(os.path.dirname(file), this_soname)))
1638 return (needs_ldconfig, needed, sonames, renames)
1639
1640 def darwin_so(file, needed, sonames, renames, pkgver):
1641 if not os.path.exists(file):
1642 return
1643 ldir = os.path.dirname(file).replace(pkgdest + "/" + pkg, '')
1644
1645 def get_combinations(base):
1646 #
1647 # Given a base library name, find all combinations of this split by "." and "-"
1648 #
1649 combos = []
1650 options = base.split(".")
1651 for i in range(1, len(options) + 1):
1652 combos.append(".".join(options[0:i]))
1653 options = base.split("-")
1654 for i in range(1, len(options) + 1):
1655 combos.append("-".join(options[0:i]))
1656 return combos
1657
1658 if (file.endswith('.dylib') or file.endswith('.so')) and not pkg.endswith('-dev') and not pkg.endswith('-dbg') and not pkg.endswith('-src'):
1659 # Drop suffix
1660 name = os.path.basename(file).rsplit(".",1)[0]
1661 # Find all combinations
1662 combos = get_combinations(name)
1663 for combo in combos:
1664 if not combo in sonames:
1665 prov = (combo, ldir, pkgver)
1666 sonames.add(prov)
1667 if file.endswith('.dylib') or file.endswith('.so'):
1668 rpath = []
1669 p = subprocess.Popen([d.expand("${HOST_PREFIX}otool"), '-l', file], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
1670 out, err = p.communicate()
1671 # If returned successfully, process stdout for results
1672 if p.returncode == 0:
1673 for l in out.split("\n"):
1674 l = l.strip()
1675 if l.startswith('path '):
1676 rpath.append(l.split()[1])
1677
1678 p = subprocess.Popen([d.expand("${HOST_PREFIX}otool"), '-L', file], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
1679 out, err = p.communicate()
1680 # If returned successfully, process stdout for results
1681 if p.returncode == 0:
1682 for l in out.split("\n"):
1683 l = l.strip()
1684 if not l or l.endswith(":"):
1685 continue
1686 if "is not an object file" in l:
1687 continue
1688 name = os.path.basename(l.split()[0]).rsplit(".", 1)[0]
1689 if name and name not in needed[pkg]:
1690 needed[pkg].add((name, file, tuple()))
1691
1692 def mingw_dll(file, needed, sonames, renames, pkgver):
1693 if not os.path.exists(file):
1694 return
1695
1696 if file.endswith(".dll"):
1697 # assume all dlls are shared objects provided by the package
1698 sonames.add((os.path.basename(file), os.path.dirname(file).replace(pkgdest + "/" + pkg, ''), pkgver))
1699
1700 if (file.endswith(".dll") or file.endswith(".exe")):
1701 # use objdump to search for "DLL Name: .*\.dll"
1702 p = subprocess.Popen([d.expand("${OBJDUMP}"), "-p", file], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
1703 out, err = p.communicate()
1704 # process the output, grabbing all .dll names
1705 if p.returncode == 0:
1706 for m in re.finditer(r"DLL Name: (.*?\.dll)$", out.decode(), re.MULTILINE | re.IGNORECASE):
1707 dllname = m.group(1)
1708 if dllname:
1709 needed[pkg].add((dllname, file, tuple()))
1710
1711 if d.getVar('PACKAGE_SNAP_LIB_SYMLINKS') == "1":
1712 snap_symlinks = True
1713 else:
1714 snap_symlinks = False
1715
1716 needed = {}
1717
1718 shlib_provider = oe.package.read_shlib_providers(d)
1719
1720 for pkg in shlib_pkgs:
1721 private_libs = d.getVar('PRIVATE_LIBS:' + pkg) or d.getVar('PRIVATE_LIBS') or ""
1722 private_libs = private_libs.split()
1723 needs_ldconfig = False
1724 bb.debug(2, "calculating shlib provides for %s" % pkg)
1725
1726 pkgver = d.getVar('PKGV:' + pkg)
1727 if not pkgver:
1728 pkgver = d.getVar('PV_' + pkg)
1729 if not pkgver:
1730 pkgver = ver
1731
1732 needed[pkg] = set()
1733 sonames = set()
1734 renames = []
1735 linuxlist = []
1736 for file in pkgfiles[pkg]:
1737 soname = None
1738 if cpath.islink(file):
1739 continue
1740 if hostos.startswith("darwin"):
1741 darwin_so(file, needed, sonames, renames, pkgver)
1742 elif hostos.startswith("mingw"):
1743 mingw_dll(file, needed, sonames, renames, pkgver)
1744 elif os.access(file, os.X_OK) or lib_re.match(file):
1745 linuxlist.append(file)
1746
1747 if linuxlist:
1748 results = oe.utils.multiprocess_launch(linux_so, linuxlist, d, extraargs=(pkg, pkgver, d))
1749 for r in results:
1750 ldconfig = r[0]
1751 needed[pkg] |= r[1]
1752 sonames |= r[2]
1753 renames.extend(r[3])
1754 needs_ldconfig = needs_ldconfig or ldconfig
1755
1756 for (old, new) in renames:
1757 bb.note("Renaming %s to %s" % (old, new))
1758 bb.utils.rename(old, new)
1759 pkgfiles[pkg].remove(old)
1760
1761 shlibs_file = os.path.join(shlibswork_dir, pkg + ".list")
1762 if len(sonames):
1763 with open(shlibs_file, 'w') as fd:
1764 for s in sorted(sonames):
1765 if s[0] in shlib_provider and s[1] in shlib_provider[s[0]]:
1766 (old_pkg, old_pkgver) = shlib_provider[s[0]][s[1]]
1767 if old_pkg != pkg:
1768 bb.warn('%s-%s was registered as shlib provider for %s, changing it to %s-%s because it was built later' % (old_pkg, old_pkgver, s[0], pkg, pkgver))
1769 bb.debug(1, 'registering %s-%s as shlib provider for %s' % (pkg, pkgver, s[0]))
1770 fd.write(s[0] + ':' + s[1] + ':' + s[2] + '\n')
1771 if s[0] not in shlib_provider:
1772 shlib_provider[s[0]] = {}
1773 shlib_provider[s[0]][s[1]] = (pkg, pkgver)
1774 if needs_ldconfig:
1775 bb.debug(1, 'adding ldconfig call to postinst for %s' % pkg)
1776 postinst = d.getVar('pkg_postinst:%s' % pkg)
1777 if not postinst:
1778 postinst = '#!/bin/sh\n'
1779 postinst += d.getVar('ldconfig_postinst_fragment')
1780 d.setVar('pkg_postinst:%s' % pkg, postinst)
1781 bb.debug(1, 'LIBNAMES: pkg %s sonames %s' % (pkg, sonames))
1782
1783 assumed_libs = d.getVar('ASSUME_SHLIBS')
1784 if assumed_libs:
1785 libdir = d.getVar("libdir")
1786 for e in assumed_libs.split():
1787 l, dep_pkg = e.split(":")
1788 lib_ver = None
1789 dep_pkg = dep_pkg.rsplit("_", 1)
1790 if len(dep_pkg) == 2:
1791 lib_ver = dep_pkg[1]
1792 dep_pkg = dep_pkg[0]
1793 if l not in shlib_provider:
1794 shlib_provider[l] = {}
1795 shlib_provider[l][libdir] = (dep_pkg, lib_ver)
1796
1797 libsearchpath = [d.getVar('libdir'), d.getVar('base_libdir')]
1798
1799 for pkg in shlib_pkgs:
1800 bb.debug(2, "calculating shlib requirements for %s" % pkg)
1801
1802 private_libs = d.getVar('PRIVATE_LIBS:' + pkg) or d.getVar('PRIVATE_LIBS') or ""
1803 private_libs = private_libs.split()
1804
1805 deps = list()
1806 for n in needed[pkg]:
1807 # if n is in private libraries, don't try to search provider for it
1808 # this could cause problem in case some abc.bb provides private
1809 # /opt/abc/lib/libfoo.so.1 and contains /usr/bin/abc depending on system library libfoo.so.1
1810 # but skipping it is still better alternative than providing own
1811 # version and then adding runtime dependency for the same system library
1812 if private_libs and len([i for i in private_libs if fnmatch.fnmatch(n[0], i)]) > 0:
1813 bb.debug(2, '%s: Dependency %s covered by PRIVATE_LIBS' % (pkg, n[0]))
1814 continue
1815 if n[0] in shlib_provider.keys():
1816 shlib_provider_map = shlib_provider[n[0]]
1817 matches = set()
1818 for p in itertools.chain(list(n[2]), sorted(shlib_provider_map.keys()), libsearchpath):
1819 if p in shlib_provider_map:
1820 matches.add(p)
1821 if len(matches) > 1:
1822 matchpkgs = ', '.join([shlib_provider_map[match][0] for match in matches])
1823 bb.error("%s: Multiple shlib providers for %s: %s (used by files: %s)" % (pkg, n[0], matchpkgs, n[1]))
1824 elif len(matches) == 1:
1825 (dep_pkg, ver_needed) = shlib_provider_map[matches.pop()]
1826
1827 bb.debug(2, '%s: Dependency %s requires package %s (used by files: %s)' % (pkg, n[0], dep_pkg, n[1]))
1828
1829 if dep_pkg == pkg:
1830 continue
1831
1832 if ver_needed:
1833 dep = "%s (>= %s)" % (dep_pkg, ver_needed)
1834 else:
1835 dep = dep_pkg
1836 if not dep in deps:
1837 deps.append(dep)
1838 continue
1839 bb.note("Couldn't find shared library provider for %s, used by files: %s" % (n[0], n[1]))
1840
1841 deps_file = os.path.join(pkgdest, pkg + ".shlibdeps")
1842 if os.path.exists(deps_file):
1843 os.remove(deps_file)
1844 if deps:
1845 with open(deps_file, 'w') as fd:
1846 for dep in sorted(deps):
1847 fd.write(dep + '\n')
1848
1849def process_pkgconfig(pkgfiles, d):
1850 packages = d.getVar('PACKAGES')
1851 workdir = d.getVar('WORKDIR')
1852 pkgdest = d.getVar('PKGDEST')
1853
1854 shlibs_dirs = d.getVar('SHLIBSDIRS').split()
1855 shlibswork_dir = d.getVar('SHLIBSWORKDIR')
1856
1857 pc_re = re.compile(r'(.*)\.pc$')
1858 var_re = re.compile(r'(.*)=(.*)')
1859 field_re = re.compile(r'(.*): (.*)')
1860
1861 pkgconfig_provided = {}
1862 pkgconfig_needed = {}
1863 for pkg in packages.split():
1864 pkgconfig_provided[pkg] = []
1865 pkgconfig_needed[pkg] = []
1866 for file in sorted(pkgfiles[pkg]):
1867 m = pc_re.match(file)
1868 if m:
1869 pd = bb.data.init()
1870 name = m.group(1)
1871 pkgconfig_provided[pkg].append(os.path.basename(name))
1872 if not os.access(file, os.R_OK):
1873 continue
1874 with open(file, 'r') as f:
1875 lines = f.readlines()
1876 for l in lines:
1877 m = field_re.match(l)
1878 if m:
1879 hdr = m.group(1)
1880 exp = pd.expand(m.group(2))
1881 if hdr == 'Requires':
1882 pkgconfig_needed[pkg] += exp.replace(',', ' ').split()
1883 continue
1884 m = var_re.match(l)
1885 if m:
1886 name = m.group(1)
1887 val = m.group(2)
1888 pd.setVar(name, pd.expand(val))
1889
1890 for pkg in packages.split():
1891 pkgs_file = os.path.join(shlibswork_dir, pkg + ".pclist")
1892 if pkgconfig_provided[pkg] != []:
1893 with open(pkgs_file, 'w') as f:
1894 for p in sorted(pkgconfig_provided[pkg]):
1895 f.write('%s\n' % p)
1896
1897 # Go from least to most specific since the last one found wins
1898 for dir in reversed(shlibs_dirs):
1899 if not os.path.exists(dir):
1900 continue
1901 for file in sorted(os.listdir(dir)):
1902 m = re.match(r'^(.*)\.pclist$', file)
1903 if m:
1904 pkg = m.group(1)
1905 with open(os.path.join(dir, file)) as fd:
1906 lines = fd.readlines()
1907 pkgconfig_provided[pkg] = []
1908 for l in lines:
1909 pkgconfig_provided[pkg].append(l.rstrip())
1910
1911 for pkg in packages.split():
1912 deps = []
1913 for n in pkgconfig_needed[pkg]:
1914 found = False
1915 for k in pkgconfig_provided.keys():
1916 if n in pkgconfig_provided[k]:
1917 if k != pkg and not (k in deps):
1918 deps.append(k)
1919 found = True
1920 if found == False:
1921 bb.note("couldn't find pkgconfig module '%s' in any package" % n)
1922 deps_file = os.path.join(pkgdest, pkg + ".pcdeps")
1923 if len(deps):
1924 with open(deps_file, 'w') as fd:
1925 for dep in deps:
1926 fd.write(dep + '\n')
1927
1928def read_libdep_files(d):
1929 pkglibdeps = {}
1930 packages = d.getVar('PACKAGES').split()
1931 for pkg in packages:
1932 pkglibdeps[pkg] = {}
1933 for extension in ".shlibdeps", ".pcdeps", ".clilibdeps":
1934 depsfile = d.expand("${PKGDEST}/" + pkg + extension)
1935 if os.access(depsfile, os.R_OK):
1936 with open(depsfile) as fd:
1937 lines = fd.readlines()
1938 for l in lines:
1939 l.rstrip()
1940 deps = bb.utils.explode_dep_versions2(l)
1941 for dep in deps:
1942 if not dep in pkglibdeps[pkg]:
1943 pkglibdeps[pkg][dep] = deps[dep]
1944 return pkglibdeps
1945
1946def process_depchains(pkgfiles, d):
1947 """
1948 For a given set of prefix and postfix modifiers, make those packages
1949 RRECOMMENDS on the corresponding packages for its RDEPENDS.
1950
1951 Example: If package A depends upon package B, and A's .bb emits an
1952 A-dev package, this would make A-dev Recommends: B-dev.
1953
1954 If only one of a given suffix is specified, it will take the RRECOMMENDS
1955 based on the RDEPENDS of *all* other packages. If more than one of a given
1956 suffix is specified, its will only use the RDEPENDS of the single parent
1957 package.
1958 """
1959
1960 packages = d.getVar('PACKAGES')
1961 postfixes = (d.getVar('DEPCHAIN_POST') or '').split()
1962 prefixes = (d.getVar('DEPCHAIN_PRE') or '').split()
1963
1964 def pkg_adddeprrecs(pkg, base, suffix, getname, depends, d):
1965
1966 #bb.note('depends for %s is %s' % (base, depends))
1967 rreclist = bb.utils.explode_dep_versions2(d.getVar('RRECOMMENDS:' + pkg) or "")
1968
1969 for depend in sorted(depends):
1970 if depend.find('-native') != -1 or depend.find('-cross') != -1 or depend.startswith('virtual/'):
1971 #bb.note("Skipping %s" % depend)
1972 continue
1973 if depend.endswith('-dev'):
1974 depend = depend[:-4]
1975 if depend.endswith('-dbg'):
1976 depend = depend[:-4]
1977 pkgname = getname(depend, suffix)
1978 #bb.note("Adding %s for %s" % (pkgname, depend))
1979 if pkgname not in rreclist and pkgname != pkg:
1980 rreclist[pkgname] = []
1981
1982 #bb.note('setting: RRECOMMENDS:%s=%s' % (pkg, ' '.join(rreclist)))
1983 d.setVar('RRECOMMENDS:%s' % pkg, bb.utils.join_deps(rreclist, commasep=False))
1984
1985 def pkg_addrrecs(pkg, base, suffix, getname, rdepends, d):
1986
1987 #bb.note('rdepends for %s is %s' % (base, rdepends))
1988 rreclist = bb.utils.explode_dep_versions2(d.getVar('RRECOMMENDS:' + pkg) or "")
1989
1990 for depend in sorted(rdepends):
1991 if depend.find('virtual-locale-') != -1:
1992 #bb.note("Skipping %s" % depend)
1993 continue
1994 if depend.endswith('-dev'):
1995 depend = depend[:-4]
1996 if depend.endswith('-dbg'):
1997 depend = depend[:-4]
1998 pkgname = getname(depend, suffix)
1999 #bb.note("Adding %s for %s" % (pkgname, depend))
2000 if pkgname not in rreclist and pkgname != pkg:
2001 rreclist[pkgname] = []
2002
2003 #bb.note('setting: RRECOMMENDS:%s=%s' % (pkg, ' '.join(rreclist)))
2004 d.setVar('RRECOMMENDS:%s' % pkg, bb.utils.join_deps(rreclist, commasep=False))
2005
2006 def add_dep(list, dep):
2007 if dep not in list:
2008 list.append(dep)
2009
2010 depends = []
2011 for dep in bb.utils.explode_deps(d.getVar('DEPENDS') or ""):
2012 add_dep(depends, dep)
2013
2014 rdepends = []
2015 for pkg in packages.split():
2016 for dep in bb.utils.explode_deps(d.getVar('RDEPENDS:' + pkg) or ""):
2017 add_dep(rdepends, dep)
2018
2019 #bb.note('rdepends is %s' % rdepends)
2020
2021 def post_getname(name, suffix):
2022 return '%s%s' % (name, suffix)
2023 def pre_getname(name, suffix):
2024 return '%s%s' % (suffix, name)
2025
2026 pkgs = {}
2027 for pkg in packages.split():
2028 for postfix in postfixes:
2029 if pkg.endswith(postfix):
2030 if not postfix in pkgs:
2031 pkgs[postfix] = {}
2032 pkgs[postfix][pkg] = (pkg[:-len(postfix)], post_getname)
2033
2034 for prefix in prefixes:
2035 if pkg.startswith(prefix):
2036 if not prefix in pkgs:
2037 pkgs[prefix] = {}
2038 pkgs[prefix][pkg] = (pkg[:-len(prefix)], pre_getname)
2039
2040 if "-dbg" in pkgs:
2041 pkglibdeps = read_libdep_files(d)
2042 pkglibdeplist = []
2043 for pkg in pkglibdeps:
2044 for k in pkglibdeps[pkg]:
2045 add_dep(pkglibdeplist, k)
2046 dbgdefaultdeps = ((d.getVar('DEPCHAIN_DBGDEFAULTDEPS') == '1') or (bb.data.inherits_class('packagegroup', d)))
2047
2048 for suffix in pkgs:
2049 for pkg in pkgs[suffix]:
2050 if d.getVarFlag('RRECOMMENDS:' + pkg, 'nodeprrecs'):
2051 continue
2052 (base, func) = pkgs[suffix][pkg]
2053 if suffix == "-dev":
2054 pkg_adddeprrecs(pkg, base, suffix, func, depends, d)
2055 elif suffix == "-dbg":
2056 if not dbgdefaultdeps:
2057 pkg_addrrecs(pkg, base, suffix, func, pkglibdeplist, d)
2058 continue
2059 if len(pkgs[suffix]) == 1:
2060 pkg_addrrecs(pkg, base, suffix, func, rdepends, d)
2061 else:
2062 rdeps = []
2063 for dep in bb.utils.explode_deps(d.getVar('RDEPENDS:' + base) or ""):
2064 add_dep(rdeps, dep)
2065 pkg_addrrecs(pkg, base, suffix, func, rdeps, d)
diff --git a/meta-xilinx-core/lib/oe/packagedata.py b/meta-xilinx-core/lib/oe/packagedata.py
new file mode 100644
index 00000000..2d1d6dde
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/packagedata.py
@@ -0,0 +1,366 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import codecs
8import os
9import json
10import bb.compress.zstd
11import oe.path
12
13from glob import glob
14
15def packaged(pkg, d):
16 return os.access(get_subpkgedata_fn(pkg, d) + '.packaged', os.R_OK)
17
18def read_pkgdatafile(fn):
19 pkgdata = {}
20
21 def decode(str):
22 c = codecs.getdecoder("unicode_escape")
23 return c(str)[0]
24
25 if os.access(fn, os.R_OK):
26 import re
27 with open(fn, 'r') as f:
28 lines = f.readlines()
29 r = re.compile(r"(^.+?):\s+(.*)")
30 for l in lines:
31 m = r.match(l)
32 if m:
33 pkgdata[m.group(1)] = decode(m.group(2))
34
35 return pkgdata
36
37def get_subpkgedata_fn(pkg, d):
38 return d.expand('${PKGDATA_DIR}/runtime/%s' % pkg)
39
40def has_subpkgdata(pkg, d):
41 return os.access(get_subpkgedata_fn(pkg, d), os.R_OK)
42
43def read_subpkgdata(pkg, d):
44 return read_pkgdatafile(get_subpkgedata_fn(pkg, d))
45
46def has_pkgdata(pn, d):
47 fn = d.expand('${PKGDATA_DIR}/%s' % pn)
48 return os.access(fn, os.R_OK)
49
50def read_pkgdata(pn, d):
51 fn = d.expand('${PKGDATA_DIR}/%s' % pn)
52 return read_pkgdatafile(fn)
53
54#
55# Collapse FOO:pkg variables into FOO
56#
57def read_subpkgdata_dict(pkg, d):
58 ret = {}
59 subd = read_pkgdatafile(get_subpkgedata_fn(pkg, d))
60 for var in subd:
61 newvar = var.replace(":" + pkg, "")
62 if newvar == var and var + ":" + pkg in subd:
63 continue
64 ret[newvar] = subd[var]
65 return ret
66
67def read_subpkgdata_extended(pkg, d):
68 import json
69 import bb.compress.zstd
70
71 fn = d.expand("${PKGDATA_DIR}/extended/%s.json.zstd" % pkg)
72 try:
73 num_threads = int(d.getVar("BB_NUMBER_THREADS"))
74 with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f:
75 return json.load(f)
76 except FileNotFoundError:
77 return None
78
79def _pkgmap(d):
80 """Return a dictionary mapping package to recipe name."""
81
82 pkgdatadir = d.getVar("PKGDATA_DIR")
83
84 pkgmap = {}
85 try:
86 files = os.listdir(pkgdatadir)
87 except OSError:
88 bb.warn("No files in %s?" % pkgdatadir)
89 files = []
90
91 for pn in [f for f in files if not os.path.isdir(os.path.join(pkgdatadir, f))]:
92 try:
93 pkgdata = read_pkgdatafile(os.path.join(pkgdatadir, pn))
94 except OSError:
95 continue
96
97 packages = pkgdata.get("PACKAGES") or ""
98 for pkg in packages.split():
99 pkgmap[pkg] = pn
100
101 return pkgmap
102
103def pkgmap(d):
104 """Return a dictionary mapping package to recipe name.
105 Cache the mapping in the metadata"""
106
107 pkgmap_data = d.getVar("__pkgmap_data", False)
108 if pkgmap_data is None:
109 pkgmap_data = _pkgmap(d)
110 d.setVar("__pkgmap_data", pkgmap_data)
111
112 return pkgmap_data
113
114def recipename(pkg, d):
115 """Return the recipe name for the given binary package name."""
116
117 return pkgmap(d).get(pkg)
118
119def foreach_runtime_provider_pkgdata(d, rdep, include_rdep=False):
120 pkgdata_dir = d.getVar("PKGDATA_DIR")
121 possibles = set()
122 try:
123 possibles |= set(os.listdir("%s/runtime-rprovides/%s/" % (pkgdata_dir, rdep)))
124 except OSError:
125 pass
126
127 if include_rdep:
128 possibles.add(rdep)
129
130 for p in sorted(list(possibles)):
131 rdep_data = read_subpkgdata(p, d)
132 yield p, rdep_data
133
134def get_package_mapping(pkg, basepkg, d, depversions=None):
135 import oe.packagedata
136
137 data = oe.packagedata.read_subpkgdata(pkg, d)
138 key = "PKG:%s" % pkg
139
140 if key in data:
141 if bb.data.inherits_class('allarch', d) and bb.data.inherits_class('packagegroup', d) and pkg != data[key]:
142 bb.error("An allarch packagegroup shouldn't depend on packages which are dynamically renamed (%s to %s)" % (pkg, data[key]))
143 # Have to avoid undoing the write_extra_pkgs(global_variants...)
144 if bb.data.inherits_class('allarch', d) and not d.getVar('MULTILIB_VARIANTS') \
145 and data[key] == basepkg:
146 return pkg
147 if depversions == []:
148 # Avoid returning a mapping if the renamed package rprovides its original name
149 rprovkey = "RPROVIDES:%s" % pkg
150 if rprovkey in data:
151 if pkg in bb.utils.explode_dep_versions2(data[rprovkey]):
152 bb.note("%s rprovides %s, not replacing the latter" % (data[key], pkg))
153 return pkg
154 # Do map to rewritten package name
155 return data[key]
156
157 return pkg
158
159def get_package_additional_metadata(pkg_type, d):
160 base_key = "PACKAGE_ADD_METADATA"
161 for key in ("%s_%s" % (base_key, pkg_type.upper()), base_key):
162 if d.getVar(key, False) is None:
163 continue
164 d.setVarFlag(key, "type", "list")
165 if d.getVarFlag(key, "separator") is None:
166 d.setVarFlag(key, "separator", "\\n")
167 metadata_fields = [field.strip() for field in oe.data.typed_value(key, d)]
168 return "\n".join(metadata_fields).strip()
169
170def runtime_mapping_rename(varname, pkg, d):
171 #bb.note("%s before: %s" % (varname, d.getVar(varname)))
172
173 new_depends = {}
174 deps = bb.utils.explode_dep_versions2(d.getVar(varname) or "")
175 for depend, depversions in deps.items():
176 new_depend = get_package_mapping(depend, pkg, d, depversions)
177 if depend != new_depend:
178 bb.note("package name mapping done: %s -> %s" % (depend, new_depend))
179 new_depends[new_depend] = deps[depend]
180
181 d.setVar(varname, bb.utils.join_deps(new_depends, commasep=False))
182
183 #bb.note("%s after: %s" % (varname, d.getVar(varname)))
184
185def emit_pkgdata(pkgfiles, d):
186 def process_postinst_on_target(pkg, mlprefix):
187 pkgval = d.getVar('PKG:%s' % pkg)
188 if pkgval is None:
189 pkgval = pkg
190
191 defer_fragment = """
192if [ -n "$D" ]; then
193 $INTERCEPT_DIR/postinst_intercept delay_to_first_boot %s mlprefix=%s
194 exit 0
195fi
196""" % (pkgval, mlprefix)
197
198 postinst = d.getVar('pkg_postinst:%s' % pkg)
199 postinst_ontarget = d.getVar('pkg_postinst_ontarget:%s' % pkg)
200
201 if postinst_ontarget:
202 bb.debug(1, 'adding deferred pkg_postinst_ontarget() to pkg_postinst() for %s' % pkg)
203 if not postinst:
204 postinst = '#!/bin/sh\n'
205 postinst += defer_fragment
206 postinst += postinst_ontarget
207 d.setVar('pkg_postinst:%s' % pkg, postinst)
208
209 def add_set_e_to_scriptlets(pkg):
210 for scriptlet_name in ('pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm'):
211 scriptlet = d.getVar('%s:%s' % (scriptlet_name, pkg))
212 if scriptlet:
213 scriptlet_split = scriptlet.split('\n')
214 if scriptlet_split[0].startswith("#!"):
215 scriptlet = scriptlet_split[0] + "\nset -e\n" + "\n".join(scriptlet_split[1:])
216 else:
217 scriptlet = "set -e\n" + "\n".join(scriptlet_split[0:])
218 d.setVar('%s:%s' % (scriptlet_name, pkg), scriptlet)
219
220 def write_if_exists(f, pkg, var):
221 def encode(str):
222 import codecs
223 c = codecs.getencoder("unicode_escape")
224 return c(str)[0].decode("latin1")
225
226 val = d.getVar('%s:%s' % (var, pkg))
227 if val:
228 f.write('%s:%s: %s\n' % (var, pkg, encode(val)))
229 return val
230 val = d.getVar('%s' % (var))
231 if val:
232 f.write('%s: %s\n' % (var, encode(val)))
233 return val
234
235 def write_extra_pkgs(variants, pn, packages, pkgdatadir):
236 for variant in variants:
237 with open("%s/%s-%s" % (pkgdatadir, variant, pn), 'w') as fd:
238 fd.write("PACKAGES: %s\n" % ' '.join(
239 map(lambda pkg: '%s-%s' % (variant, pkg), packages.split())))
240
241 def write_extra_runtime_pkgs(variants, packages, pkgdatadir):
242 for variant in variants:
243 for pkg in packages.split():
244 ml_pkg = "%s-%s" % (variant, pkg)
245 subdata_file = "%s/runtime/%s" % (pkgdatadir, ml_pkg)
246 with open(subdata_file, 'w') as fd:
247 fd.write("PKG:%s: %s" % (ml_pkg, pkg))
248
249 packages = d.getVar('PACKAGES')
250 pkgdest = d.getVar('PKGDEST')
251 pkgdatadir = d.getVar('PKGDESTWORK')
252
253 data_file = pkgdatadir + d.expand("/${PN}")
254 with open(data_file, 'w') as fd:
255 fd.write("PACKAGES: %s\n" % packages)
256
257 pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or []
258
259 pn = d.getVar('PN')
260 global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split()
261 variants = (d.getVar('MULTILIB_VARIANTS') or "").split()
262
263 if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d):
264 write_extra_pkgs(variants, pn, packages, pkgdatadir)
265
266 if bb.data.inherits_class('allarch', d) and not variants \
267 and not bb.data.inherits_class('packagegroup', d):
268 write_extra_pkgs(global_variants, pn, packages, pkgdatadir)
269
270 workdir = d.getVar('WORKDIR')
271
272 for pkg in packages.split():
273 pkgval = d.getVar('PKG:%s' % pkg)
274 if pkgval is None:
275 pkgval = pkg
276 d.setVar('PKG:%s' % pkg, pkg)
277
278 extended_data = {
279 "files_info": {}
280 }
281
282 pkgdestpkg = os.path.join(pkgdest, pkg)
283 files = {}
284 files_extra = {}
285 total_size = 0
286 seen = set()
287 for f in pkgfiles[pkg]:
288 fpath = os.sep + os.path.relpath(f, pkgdestpkg)
289
290 fstat = os.lstat(f)
291 files[fpath] = fstat.st_size
292
293 extended_data["files_info"].setdefault(fpath, {})
294 extended_data["files_info"][fpath]['size'] = fstat.st_size
295
296 if fstat.st_ino not in seen:
297 seen.add(fstat.st_ino)
298 total_size += fstat.st_size
299
300 if fpath in pkgdebugsource:
301 extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath]
302 del pkgdebugsource[fpath]
303
304 d.setVar('FILES_INFO:' + pkg , json.dumps(files, sort_keys=True))
305
306 process_postinst_on_target(pkg, d.getVar("MLPREFIX"))
307 add_set_e_to_scriptlets(pkg)
308
309 subdata_file = pkgdatadir + "/runtime/%s" % pkg
310 with open(subdata_file, 'w') as sf:
311 for var in (d.getVar('PKGDATA_VARS') or "").split():
312 val = write_if_exists(sf, pkg, var)
313
314 write_if_exists(sf, pkg, 'FILERPROVIDESFLIST')
315 for dfile in sorted((d.getVar('FILERPROVIDESFLIST:' + pkg) or "").split()):
316 write_if_exists(sf, pkg, 'FILERPROVIDES:' + dfile)
317
318 write_if_exists(sf, pkg, 'FILERDEPENDSFLIST')
319 for dfile in sorted((d.getVar('FILERDEPENDSFLIST:' + pkg) or "").split()):
320 write_if_exists(sf, pkg, 'FILERDEPENDS:' + dfile)
321
322 sf.write('%s:%s: %d\n' % ('PKGSIZE', pkg, total_size))
323
324 subdata_extended_file = pkgdatadir + "/extended/%s.json.zstd" % pkg
325 num_threads = int(d.getVar("BB_NUMBER_THREADS"))
326 with bb.compress.zstd.open(subdata_extended_file, "wt", encoding="utf-8", num_threads=num_threads) as f:
327 json.dump(extended_data, f, sort_keys=True, separators=(",", ":"))
328
329 # Symlinks needed for rprovides lookup
330 rprov = d.getVar('RPROVIDES:%s' % pkg) or d.getVar('RPROVIDES')
331 if rprov:
332 for p in bb.utils.explode_deps(rprov):
333 subdata_sym = pkgdatadir + "/runtime-rprovides/%s/%s" % (p, pkg)
334 bb.utils.mkdirhier(os.path.dirname(subdata_sym))
335 oe.path.relsymlink(subdata_file, subdata_sym, True)
336
337 allow_empty = d.getVar('ALLOW_EMPTY:%s' % pkg)
338 if not allow_empty:
339 allow_empty = d.getVar('ALLOW_EMPTY')
340 root = "%s/%s" % (pkgdest, pkg)
341 os.chdir(root)
342 g = glob('*')
343 if g or allow_empty == "1":
344 # Symlinks needed for reverse lookups (from the final package name)
345 subdata_sym = pkgdatadir + "/runtime-reverse/%s" % pkgval
346 oe.path.relsymlink(subdata_file, subdata_sym, True)
347
348 packagedfile = pkgdatadir + '/runtime/%s.packaged' % pkg
349 open(packagedfile, 'w').close()
350
351 if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d):
352 write_extra_runtime_pkgs(variants, packages, pkgdatadir)
353
354 if bb.data.inherits_class('allarch', d) and not variants \
355 and not bb.data.inherits_class('packagegroup', d):
356 write_extra_runtime_pkgs(global_variants, packages, pkgdatadir)
357
358def mapping_rename_hook(d):
359 """
360 Rewrite variables to account for package renaming in things
361 like debian.bbclass or manual PKG variable name changes
362 """
363 pkg = d.getVar("PKG")
364 oe.packagedata.runtime_mapping_rename("RDEPENDS", pkg, d)
365 oe.packagedata.runtime_mapping_rename("RRECOMMENDS", pkg, d)
366 oe.packagedata.runtime_mapping_rename("RSUGGESTS", pkg, d)
diff --git a/meta-xilinx-core/lib/oe/packagegroup.py b/meta-xilinx-core/lib/oe/packagegroup.py
new file mode 100644
index 00000000..7b759475
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/packagegroup.py
@@ -0,0 +1,36 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import itertools
8
9def is_optional(feature, d):
10 return bool(d.getVarFlag("FEATURE_PACKAGES_%s" % feature, "optional"))
11
12def packages(features, d):
13 for feature in features:
14 packages = d.getVar("FEATURE_PACKAGES_%s" % feature)
15 for pkg in (packages or "").split():
16 yield pkg
17
18def required_packages(features, d):
19 req = [feature for feature in features if not is_optional(feature, d)]
20 return packages(req, d)
21
22def optional_packages(features, d):
23 opt = [feature for feature in features if is_optional(feature, d)]
24 return packages(opt, d)
25
26def active_packages(features, d):
27 return itertools.chain(required_packages(features, d),
28 optional_packages(features, d))
29
30def active_recipes(features, d):
31 import oe.packagedata
32
33 for pkg in active_packages(features, d):
34 recipe = oe.packagedata.recipename(pkg, d)
35 if recipe:
36 yield recipe
diff --git a/meta-xilinx-core/lib/oe/patch.py b/meta-xilinx-core/lib/oe/patch.py
new file mode 100644
index 00000000..60a0cc82
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/patch.py
@@ -0,0 +1,1001 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import os
8import shlex
9import subprocess
10import oe.path
11import oe.types
12
13class NotFoundError(bb.BBHandledException):
14 def __init__(self, path):
15 self.path = path
16
17 def __str__(self):
18 return "Error: %s not found." % self.path
19
20class CmdError(bb.BBHandledException):
21 def __init__(self, command, exitstatus, output):
22 self.command = command
23 self.status = exitstatus
24 self.output = output
25
26 def __str__(self):
27 return "Command Error: '%s' exited with %d Output:\n%s" % \
28 (self.command, self.status, self.output)
29
30
31def runcmd(args, dir = None):
32 if dir:
33 olddir = os.path.abspath(os.curdir)
34 if not os.path.exists(dir):
35 raise NotFoundError(dir)
36 os.chdir(dir)
37 # print("cwd: %s -> %s" % (olddir, dir))
38
39 try:
40 args = [ shlex.quote(str(arg)) for arg in args ]
41 cmd = " ".join(args)
42 # print("cmd: %s" % cmd)
43 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
44 stdout, stderr = proc.communicate()
45 stdout = stdout.decode('utf-8')
46 stderr = stderr.decode('utf-8')
47 exitstatus = proc.returncode
48 if exitstatus != 0:
49 raise CmdError(cmd, exitstatus >> 8, "stdout: %s\nstderr: %s" % (stdout, stderr))
50 if " fuzz " in stdout and "Hunk " in stdout:
51 # Drop patch fuzz info with header and footer to log file so
52 # insane.bbclass can handle to throw error/warning
53 bb.note("--- Patch fuzz start ---\n%s\n--- Patch fuzz end ---" % format(stdout))
54
55 return stdout
56
57 finally:
58 if dir:
59 os.chdir(olddir)
60
61
62class PatchError(Exception):
63 def __init__(self, msg):
64 self.msg = msg
65
66 def __str__(self):
67 return "Patch Error: %s" % self.msg
68
69class PatchSet(object):
70 defaults = {
71 "strippath": 1
72 }
73
74 def __init__(self, dir, d):
75 self.dir = dir
76 self.d = d
77 self.patches = []
78 self._current = None
79
80 def current(self):
81 return self._current
82
83 def Clean(self):
84 """
85 Clean out the patch set. Generally includes unapplying all
86 patches and wiping out all associated metadata.
87 """
88 raise NotImplementedError()
89
90 def Import(self, patch, force):
91 if not patch.get("file"):
92 if not patch.get("remote"):
93 raise PatchError("Patch file must be specified in patch import.")
94 else:
95 patch["file"] = bb.fetch2.localpath(patch["remote"], self.d)
96
97 for param in PatchSet.defaults:
98 if not patch.get(param):
99 patch[param] = PatchSet.defaults[param]
100
101 if patch.get("remote"):
102 patch["file"] = self.d.expand(bb.fetch2.localpath(patch["remote"], self.d))
103
104 patch["filemd5"] = bb.utils.md5_file(patch["file"])
105
106 def Push(self, force):
107 raise NotImplementedError()
108
109 def Pop(self, force):
110 raise NotImplementedError()
111
112 def Refresh(self, remote = None, all = None):
113 raise NotImplementedError()
114
115 @staticmethod
116 def getPatchedFiles(patchfile, striplevel, srcdir=None):
117 """
118 Read a patch file and determine which files it will modify.
119 Params:
120 patchfile: the patch file to read
121 striplevel: the strip level at which the patch is going to be applied
122 srcdir: optional path to join onto the patched file paths
123 Returns:
124 A list of tuples of file path and change mode ('A' for add,
125 'D' for delete or 'M' for modify)
126 """
127
128 def patchedpath(patchline):
129 filepth = patchline.split()[1]
130 if filepth.endswith('/dev/null'):
131 return '/dev/null'
132 filesplit = filepth.split(os.sep)
133 if striplevel > len(filesplit):
134 bb.error('Patch %s has invalid strip level %d' % (patchfile, striplevel))
135 return None
136 return os.sep.join(filesplit[striplevel:])
137
138 for encoding in ['utf-8', 'latin-1']:
139 try:
140 copiedmode = False
141 filelist = []
142 with open(patchfile) as f:
143 for line in f:
144 if line.startswith('--- '):
145 patchpth = patchedpath(line)
146 if not patchpth:
147 break
148 if copiedmode:
149 addedfile = patchpth
150 else:
151 removedfile = patchpth
152 elif line.startswith('+++ '):
153 addedfile = patchedpath(line)
154 if not addedfile:
155 break
156 elif line.startswith('*** '):
157 copiedmode = True
158 removedfile = patchedpath(line)
159 if not removedfile:
160 break
161 else:
162 removedfile = None
163 addedfile = None
164
165 if addedfile and removedfile:
166 if removedfile == '/dev/null':
167 mode = 'A'
168 elif addedfile == '/dev/null':
169 mode = 'D'
170 else:
171 mode = 'M'
172 if srcdir:
173 fullpath = os.path.abspath(os.path.join(srcdir, addedfile))
174 else:
175 fullpath = addedfile
176 filelist.append((fullpath, mode))
177 except UnicodeDecodeError:
178 continue
179 break
180 else:
181 raise PatchError('Unable to decode %s' % patchfile)
182
183 return filelist
184
185
186class PatchTree(PatchSet):
187 def __init__(self, dir, d):
188 PatchSet.__init__(self, dir, d)
189 self.patchdir = os.path.join(self.dir, 'patches')
190 self.seriespath = os.path.join(self.dir, 'patches', 'series')
191 bb.utils.mkdirhier(self.patchdir)
192
193 def _appendPatchFile(self, patch, strippath):
194 with open(self.seriespath, 'a') as f:
195 f.write(os.path.basename(patch) + "," + strippath + "\n")
196 shellcmd = ["cat", patch, ">" , self.patchdir + "/" + os.path.basename(patch)]
197 runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
198
199 def _removePatch(self, p):
200 patch = {}
201 patch['file'] = p.split(",")[0]
202 patch['strippath'] = p.split(",")[1]
203 self._applypatch(patch, False, True)
204
205 def _removePatchFile(self, all = False):
206 if not os.path.exists(self.seriespath):
207 return
208 with open(self.seriespath, 'r+') as f:
209 patches = f.readlines()
210 if all:
211 for p in reversed(patches):
212 self._removePatch(os.path.join(self.patchdir, p.strip()))
213 patches = []
214 else:
215 self._removePatch(os.path.join(self.patchdir, patches[-1].strip()))
216 patches.pop()
217 with open(self.seriespath, 'w') as f:
218 for p in patches:
219 f.write(p)
220
221 def Import(self, patch, force = None):
222 """"""
223 PatchSet.Import(self, patch, force)
224
225 if self._current is not None:
226 i = self._current + 1
227 else:
228 i = 0
229 self.patches.insert(i, patch)
230
231 def _applypatch(self, patch, force = False, reverse = False, run = True):
232 shellcmd = ["cat", patch['file'], "|", "patch", "--no-backup-if-mismatch", "-p", patch['strippath']]
233 if reverse:
234 shellcmd.append('-R')
235
236 if not run:
237 return "sh" + "-c" + " ".join(shellcmd)
238
239 if not force:
240 shellcmd.append('--dry-run')
241
242 try:
243 output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
244
245 if force:
246 return
247
248 shellcmd.pop(len(shellcmd) - 1)
249 output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
250 except CmdError as err:
251 raise bb.BBHandledException("Applying '%s' failed:\n%s" %
252 (os.path.basename(patch['file']), err.output))
253
254 if not reverse:
255 self._appendPatchFile(patch['file'], patch['strippath'])
256
257 return output
258
259 def Push(self, force = False, all = False, run = True):
260 bb.note("self._current is %s" % self._current)
261 bb.note("patches is %s" % self.patches)
262 if all:
263 for i in self.patches:
264 bb.note("applying patch %s" % i)
265 self._applypatch(i, force)
266 self._current = i
267 else:
268 if self._current is not None:
269 next = self._current + 1
270 else:
271 next = 0
272
273 bb.note("applying patch %s" % self.patches[next])
274 ret = self._applypatch(self.patches[next], force)
275
276 self._current = next
277 return ret
278
279 def Pop(self, force = None, all = None):
280 if all:
281 self._removePatchFile(True)
282 self._current = None
283 else:
284 self._removePatchFile(False)
285
286 if self._current == 0:
287 self._current = None
288
289 if self._current is not None:
290 self._current = self._current - 1
291
292 def Clean(self):
293 """"""
294 self.Pop(all=True)
295
296class GitApplyTree(PatchTree):
297 notes_ref = "refs/notes/devtool"
298 original_patch = 'original patch'
299 ignore_commit = 'ignore'
300
301 def __init__(self, dir, d):
302 PatchTree.__init__(self, dir, d)
303 self.commituser = d.getVar('PATCH_GIT_USER_NAME')
304 self.commitemail = d.getVar('PATCH_GIT_USER_EMAIL')
305 if not self._isInitialized(d):
306 self._initRepo()
307
308 def _isInitialized(self, d):
309 cmd = "git rev-parse --show-toplevel"
310 try:
311 output = runcmd(cmd.split(), self.dir).strip()
312 except CmdError as err:
313 ## runcmd returned non-zero which most likely means 128
314 ## Not a git directory
315 return False
316 ## Make sure repo is in builddir to not break top-level git repos, or under workdir
317 return os.path.samefile(output, self.dir) or oe.path.is_path_parent(d.getVar('WORKDIR'), output)
318
319 def _initRepo(self):
320 runcmd("git init".split(), self.dir)
321 runcmd("git add .".split(), self.dir)
322 runcmd("git commit -a --allow-empty -m bitbake_patching_started".split(), self.dir)
323
324 @staticmethod
325 def extractPatchHeader(patchfile):
326 """
327 Extract just the header lines from the top of a patch file
328 """
329 for encoding in ['utf-8', 'latin-1']:
330 lines = []
331 try:
332 with open(patchfile, 'r', encoding=encoding) as f:
333 for line in f:
334 if line.startswith('Index: ') or line.startswith('diff -') or line.startswith('---'):
335 break
336 lines.append(line)
337 except UnicodeDecodeError:
338 continue
339 break
340 else:
341 raise PatchError('Unable to find a character encoding to decode %s' % patchfile)
342 return lines
343
344 @staticmethod
345 def decodeAuthor(line):
346 from email.header import decode_header
347 authorval = line.split(':', 1)[1].strip().replace('"', '')
348 result = decode_header(authorval)[0][0]
349 if hasattr(result, 'decode'):
350 result = result.decode('utf-8')
351 return result
352
353 @staticmethod
354 def interpretPatchHeader(headerlines):
355 import re
356 author_re = re.compile(r'[\S ]+ <\S+@\S+\.\S+>')
357 from_commit_re = re.compile(r'^From [a-z0-9]{40} .*')
358 outlines = []
359 author = None
360 date = None
361 subject = None
362 for line in headerlines:
363 if line.startswith('Subject: '):
364 subject = line.split(':', 1)[1]
365 # Remove any [PATCH][oe-core] etc.
366 subject = re.sub(r'\[.+?\]\s*', '', subject)
367 continue
368 elif line.startswith('From: ') or line.startswith('Author: '):
369 authorval = GitApplyTree.decodeAuthor(line)
370 # git is fussy about author formatting i.e. it must be Name <email@domain>
371 if author_re.match(authorval):
372 author = authorval
373 continue
374 elif line.startswith('Date: '):
375 if date is None:
376 dateval = line.split(':', 1)[1].strip()
377 # Very crude check for date format, since git will blow up if it's not in the right
378 # format. Without e.g. a python-dateutils dependency we can't do a whole lot more
379 if len(dateval) > 12:
380 date = dateval
381 continue
382 elif not author and line.lower().startswith('signed-off-by: '):
383 authorval = GitApplyTree.decodeAuthor(line)
384 # git is fussy about author formatting i.e. it must be Name <email@domain>
385 if author_re.match(authorval):
386 author = authorval
387 elif from_commit_re.match(line):
388 # We don't want the From <commit> line - if it's present it will break rebasing
389 continue
390 outlines.append(line)
391
392 if not subject:
393 firstline = None
394 for line in headerlines:
395 line = line.strip()
396 if firstline:
397 if line:
398 # Second line is not blank, the first line probably isn't usable
399 firstline = None
400 break
401 elif line:
402 firstline = line
403 if firstline and not firstline.startswith(('#', 'Index:', 'Upstream-Status:')) and len(firstline) < 100:
404 subject = firstline
405
406 return outlines, author, date, subject
407
408 @staticmethod
409 def gitCommandUserOptions(cmd, commituser=None, commitemail=None, d=None):
410 if d:
411 commituser = d.getVar('PATCH_GIT_USER_NAME')
412 commitemail = d.getVar('PATCH_GIT_USER_EMAIL')
413 if commituser:
414 cmd += ['-c', 'user.name="%s"' % commituser]
415 if commitemail:
416 cmd += ['-c', 'user.email="%s"' % commitemail]
417
418 @staticmethod
419 def prepareCommit(patchfile, commituser=None, commitemail=None):
420 """
421 Prepare a git commit command line based on the header from a patch file
422 (typically this is useful for patches that cannot be applied with "git am" due to formatting)
423 """
424 import tempfile
425 # Process patch header and extract useful information
426 lines = GitApplyTree.extractPatchHeader(patchfile)
427 outlines, author, date, subject = GitApplyTree.interpretPatchHeader(lines)
428 if not author or not subject or not date:
429 try:
430 shellcmd = ["git", "log", "--format=email", "--follow", "--diff-filter=A", "--", patchfile]
431 out = runcmd(["sh", "-c", " ".join(shellcmd)], os.path.dirname(patchfile))
432 except CmdError:
433 out = None
434 if out:
435 _, newauthor, newdate, newsubject = GitApplyTree.interpretPatchHeader(out.splitlines())
436 if not author:
437 # If we're setting the author then the date should be set as well
438 author = newauthor
439 date = newdate
440 elif not date:
441 # If we don't do this we'll get the current date, at least this will be closer
442 date = newdate
443 if not subject:
444 subject = newsubject
445 if subject and not (outlines and outlines[0].strip() == subject):
446 outlines.insert(0, '%s\n\n' % subject.strip())
447
448 # Write out commit message to a file
449 with tempfile.NamedTemporaryFile('w', delete=False) as tf:
450 tmpfile = tf.name
451 for line in outlines:
452 tf.write(line)
453 # Prepare git command
454 cmd = ["git"]
455 GitApplyTree.gitCommandUserOptions(cmd, commituser, commitemail)
456 cmd += ["commit", "-F", tmpfile, "--no-verify"]
457 # git doesn't like plain email addresses as authors
458 if author and '<' in author:
459 cmd.append('--author="%s"' % author)
460 if date:
461 cmd.append('--date="%s"' % date)
462 return (tmpfile, cmd)
463
464 @staticmethod
465 def addNote(repo, ref, key, value=None):
466 note = key + (": %s" % value if value else "")
467 notes_ref = GitApplyTree.notes_ref
468 runcmd(["git", "config", "notes.rewriteMode", "ignore"], repo)
469 runcmd(["git", "config", "notes.displayRef", notes_ref, notes_ref], repo)
470 runcmd(["git", "config", "notes.rewriteRef", notes_ref, notes_ref], repo)
471 runcmd(["git", "notes", "--ref", notes_ref, "append", "-m", note, ref], repo)
472
473 @staticmethod
474 def removeNote(repo, ref, key):
475 notes = GitApplyTree.getNotes(repo, ref)
476 notes = {k: v for k, v in notes.items() if k != key and not k.startswith(key + ":")}
477 runcmd(["git", "notes", "--ref", GitApplyTree.notes_ref, "remove", "--ignore-missing", ref], repo)
478 for note, value in notes.items():
479 GitApplyTree.addNote(repo, ref, note, value)
480
481 @staticmethod
482 def getNotes(repo, ref):
483 import re
484
485 note = None
486 try:
487 note = runcmd(["git", "notes", "--ref", GitApplyTree.notes_ref, "show", ref], repo)
488 prefix = ""
489 except CmdError:
490 note = runcmd(['git', 'show', '-s', '--format=%B', ref], repo)
491 prefix = "%% "
492
493 note_re = re.compile(r'^%s(.*?)(?::\s*(.*))?$' % prefix)
494 notes = dict()
495 for line in note.splitlines():
496 m = note_re.match(line)
497 if m:
498 notes[m.group(1)] = m.group(2)
499
500 return notes
501
502 @staticmethod
503 def commitIgnored(subject, dir=None, files=None, d=None):
504 if files:
505 runcmd(['git', 'add'] + files, dir)
506 cmd = ["git"]
507 GitApplyTree.gitCommandUserOptions(cmd, d=d)
508 cmd += ["commit", "-m", subject, "--no-verify"]
509 runcmd(cmd, dir)
510 GitApplyTree.addNote(dir, "HEAD", GitApplyTree.ignore_commit)
511
512 @staticmethod
513 def extractPatches(tree, startcommits, outdir, paths=None):
514 import tempfile
515 import shutil
516 tempdir = tempfile.mkdtemp(prefix='oepatch')
517 try:
518 for name, rev in startcommits.items():
519 shellcmd = ["git", "format-patch", "--no-signature", "--no-numbered", rev, "-o", tempdir]
520 if paths:
521 shellcmd.append('--')
522 shellcmd.extend(paths)
523 out = runcmd(["sh", "-c", " ".join(shellcmd)], os.path.join(tree, name))
524 if out:
525 for srcfile in out.split():
526 # This loop, which is used to remove any line that
527 # starts with "%% original patch", is kept for backwards
528 # compatibility. If/when that compatibility is dropped,
529 # it can be replaced with code to just read the first
530 # line of the patch file to get the SHA-1, and the code
531 # below that writes the modified patch file can be
532 # replaced with a simple file move.
533 for encoding in ['utf-8', 'latin-1']:
534 patchlines = []
535 try:
536 with open(srcfile, 'r', encoding=encoding, newline='') as f:
537 for line in f:
538 if line.startswith("%% " + GitApplyTree.original_patch):
539 continue
540 patchlines.append(line)
541 except UnicodeDecodeError:
542 continue
543 break
544 else:
545 raise PatchError('Unable to find a character encoding to decode %s' % srcfile)
546
547 sha1 = patchlines[0].split()[1]
548 notes = GitApplyTree.getNotes(os.path.join(tree, name), sha1)
549 if GitApplyTree.ignore_commit in notes:
550 continue
551 outfile = notes.get(GitApplyTree.original_patch, os.path.basename(srcfile))
552
553 bb.utils.mkdirhier(os.path.join(outdir, name))
554 with open(os.path.join(outdir, name, outfile), 'w') as of:
555 for line in patchlines:
556 of.write(line)
557 finally:
558 shutil.rmtree(tempdir)
559
560 def _need_dirty_check(self):
561 fetch = bb.fetch2.Fetch([], self.d)
562 check_dirtyness = False
563 for url in fetch.urls:
564 url_data = fetch.ud[url]
565 parm = url_data.parm
566 # a git url with subpath param will surely be dirty
567 # since the git tree from which we clone will be emptied
568 # from all files that are not in the subpath
569 if url_data.type == 'git' and parm.get('subpath'):
570 check_dirtyness = True
571 return check_dirtyness
572
573 def _commitpatch(self, patch, patchfilevar):
574 output = ""
575 # Add all files
576 shellcmd = ["git", "add", "-f", "-A", "."]
577 output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
578 # Exclude the patches directory
579 shellcmd = ["git", "reset", "HEAD", self.patchdir]
580 output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
581 # Commit the result
582 (tmpfile, shellcmd) = self.prepareCommit(patch['file'], self.commituser, self.commitemail)
583 try:
584 shellcmd.insert(0, patchfilevar)
585 output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
586 finally:
587 os.remove(tmpfile)
588 return output
589
590 def _applypatch(self, patch, force = False, reverse = False, run = True):
591 import shutil
592
593 def _applypatchhelper(shellcmd, patch, force = False, reverse = False, run = True):
594 if reverse:
595 shellcmd.append('-R')
596
597 shellcmd.append(patch['file'])
598
599 if not run:
600 return "sh" + "-c" + " ".join(shellcmd)
601
602 return runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
603
604 reporoot = (runcmd("git rev-parse --show-toplevel".split(), self.dir) or '').strip()
605 if not reporoot:
606 raise Exception("Cannot get repository root for directory %s" % self.dir)
607
608 patch_applied = True
609 try:
610 patchfilevar = 'PATCHFILE="%s"' % os.path.basename(patch['file'])
611 if self._need_dirty_check():
612 # Check dirtyness of the tree
613 try:
614 output = runcmd(["git", "--work-tree=%s" % reporoot, "status", "--short"])
615 except CmdError:
616 pass
617 else:
618 if output:
619 # The tree is dirty, no need to try to apply patches with git anymore
620 # since they fail, fallback directly to patch
621 output = PatchTree._applypatch(self, patch, force, reverse, run)
622 output += self._commitpatch(patch, patchfilevar)
623 return output
624 try:
625 shellcmd = [patchfilevar, "git", "--work-tree=%s" % reporoot]
626 self.gitCommandUserOptions(shellcmd, self.commituser, self.commitemail)
627 shellcmd += ["am", "-3", "--keep-cr", "--no-scissors", "-p%s" % patch['strippath']]
628 return _applypatchhelper(shellcmd, patch, force, reverse, run)
629 except CmdError:
630 # Need to abort the git am, or we'll still be within it at the end
631 try:
632 shellcmd = ["git", "--work-tree=%s" % reporoot, "am", "--abort"]
633 runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
634 except CmdError:
635 pass
636 # git am won't always clean up after itself, sadly, so...
637 shellcmd = ["git", "--work-tree=%s" % reporoot, "reset", "--hard", "HEAD"]
638 runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
639 # Also need to take care of any stray untracked files
640 shellcmd = ["git", "--work-tree=%s" % reporoot, "clean", "-f"]
641 runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
642
643 # Fall back to git apply
644 shellcmd = ["git", "--git-dir=%s" % reporoot, "apply", "-p%s" % patch['strippath']]
645 try:
646 output = _applypatchhelper(shellcmd, patch, force, reverse, run)
647 except CmdError:
648 # Fall back to patch
649 output = PatchTree._applypatch(self, patch, force, reverse, run)
650 output += self._commitpatch(patch, patchfilevar)
651 return output
652 except:
653 patch_applied = False
654 raise
655 finally:
656 if patch_applied:
657 GitApplyTree.addNote(self.dir, "HEAD", GitApplyTree.original_patch, os.path.basename(patch['file']))
658
659
660class QuiltTree(PatchSet):
661 def _runcmd(self, args, run = True):
662 quiltrc = self.d.getVar('QUILTRCFILE')
663 if not run:
664 return ["quilt"] + ["--quiltrc"] + [quiltrc] + args
665 runcmd(["quilt"] + ["--quiltrc"] + [quiltrc] + args, self.dir)
666
667 def _quiltpatchpath(self, file):
668 return os.path.join(self.dir, "patches", os.path.basename(file))
669
670
671 def __init__(self, dir, d):
672 PatchSet.__init__(self, dir, d)
673 self.initialized = False
674 p = os.path.join(self.dir, 'patches')
675 if not os.path.exists(p):
676 os.makedirs(p)
677
678 def Clean(self):
679 try:
680 # make sure that patches/series file exists before quilt pop to keep quilt-0.67 happy
681 open(os.path.join(self.dir, "patches","series"), 'a').close()
682 self._runcmd(["pop", "-a", "-f"])
683 oe.path.remove(os.path.join(self.dir, "patches","series"))
684 except Exception:
685 pass
686 self.initialized = True
687
688 def InitFromDir(self):
689 # read series -> self.patches
690 seriespath = os.path.join(self.dir, 'patches', 'series')
691 if not os.path.exists(self.dir):
692 raise NotFoundError(self.dir)
693 if os.path.exists(seriespath):
694 with open(seriespath, 'r') as f:
695 for line in f.readlines():
696 patch = {}
697 parts = line.strip().split()
698 patch["quiltfile"] = self._quiltpatchpath(parts[0])
699 patch["quiltfilemd5"] = bb.utils.md5_file(patch["quiltfile"])
700 if len(parts) > 1:
701 patch["strippath"] = parts[1][2:]
702 self.patches.append(patch)
703
704 # determine which patches are applied -> self._current
705 try:
706 output = runcmd(["quilt", "applied"], self.dir)
707 except CmdError:
708 import sys
709 if sys.exc_value.output.strip() == "No patches applied":
710 return
711 else:
712 raise
713 output = [val for val in output.split('\n') if not val.startswith('#')]
714 for patch in self.patches:
715 if os.path.basename(patch["quiltfile"]) == output[-1]:
716 self._current = self.patches.index(patch)
717 self.initialized = True
718
719 def Import(self, patch, force = None):
720 if not self.initialized:
721 self.InitFromDir()
722 PatchSet.Import(self, patch, force)
723 oe.path.symlink(patch["file"], self._quiltpatchpath(patch["file"]), force=True)
724 with open(os.path.join(self.dir, "patches", "series"), "a") as f:
725 f.write(os.path.basename(patch["file"]) + " -p" + patch["strippath"] + "\n")
726 patch["quiltfile"] = self._quiltpatchpath(patch["file"])
727 patch["quiltfilemd5"] = bb.utils.md5_file(patch["quiltfile"])
728
729 # TODO: determine if the file being imported:
730 # 1) is already imported, and is the same
731 # 2) is already imported, but differs
732
733 self.patches.insert(self._current or 0, patch)
734
735
736 def Push(self, force = False, all = False, run = True):
737 # quilt push [-f]
738
739 args = ["push"]
740 if force:
741 args.append("-f")
742 if all:
743 args.append("-a")
744 if not run:
745 return self._runcmd(args, run)
746
747 self._runcmd(args)
748
749 if self._current is not None:
750 self._current = self._current + 1
751 else:
752 self._current = 0
753
754 def Pop(self, force = None, all = None):
755 # quilt pop [-f]
756 args = ["pop"]
757 if force:
758 args.append("-f")
759 if all:
760 args.append("-a")
761
762 self._runcmd(args)
763
764 if self._current == 0:
765 self._current = None
766
767 if self._current is not None:
768 self._current = self._current - 1
769
770 def Refresh(self, **kwargs):
771 if kwargs.get("remote"):
772 patch = self.patches[kwargs["patch"]]
773 if not patch:
774 raise PatchError("No patch found at index %s in patchset." % kwargs["patch"])
775 (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(patch["remote"])
776 if type == "file":
777 import shutil
778 if not patch.get("file") and patch.get("remote"):
779 patch["file"] = bb.fetch2.localpath(patch["remote"], self.d)
780
781 shutil.copyfile(patch["quiltfile"], patch["file"])
782 else:
783 raise PatchError("Unable to do a remote refresh of %s, unsupported remote url scheme %s." % (os.path.basename(patch["quiltfile"]), type))
784 else:
785 # quilt refresh
786 args = ["refresh"]
787 if kwargs.get("quiltfile"):
788 args.append(os.path.basename(kwargs["quiltfile"]))
789 elif kwargs.get("patch"):
790 args.append(os.path.basename(self.patches[kwargs["patch"]]["quiltfile"]))
791 self._runcmd(args)
792
793class Resolver(object):
794 def __init__(self, patchset, terminal):
795 raise NotImplementedError()
796
797 def Resolve(self):
798 raise NotImplementedError()
799
800 def Revert(self):
801 raise NotImplementedError()
802
803 def Finalize(self):
804 raise NotImplementedError()
805
806class NOOPResolver(Resolver):
807 def __init__(self, patchset, terminal):
808 self.patchset = patchset
809 self.terminal = terminal
810
811 def Resolve(self):
812 olddir = os.path.abspath(os.curdir)
813 os.chdir(self.patchset.dir)
814 try:
815 self.patchset.Push()
816 except Exception:
817 import sys
818 raise
819 finally:
820 os.chdir(olddir)
821
822# Patch resolver which relies on the user doing all the work involved in the
823# resolution, with the exception of refreshing the remote copy of the patch
824# files (the urls).
825class UserResolver(Resolver):
826 def __init__(self, patchset, terminal):
827 self.patchset = patchset
828 self.terminal = terminal
829
830 # Force a push in the patchset, then drop to a shell for the user to
831 # resolve any rejected hunks
832 def Resolve(self):
833 olddir = os.path.abspath(os.curdir)
834 os.chdir(self.patchset.dir)
835 try:
836 self.patchset.Push(False)
837 except CmdError as v:
838 # Patch application failed
839 patchcmd = self.patchset.Push(True, False, False)
840
841 t = self.patchset.d.getVar('T')
842 if not t:
843 bb.msg.fatal("Build", "T not set")
844 bb.utils.mkdirhier(t)
845 import random
846 rcfile = "%s/bashrc.%s.%s" % (t, str(os.getpid()), random.random())
847 with open(rcfile, "w") as f:
848 f.write("echo '*** Manual patch resolution mode ***'\n")
849 f.write("echo 'Dropping to a shell, so patch rejects can be fixed manually.'\n")
850 f.write("echo 'Run \"quilt refresh\" when patch is corrected, press CTRL+D to exit.'\n")
851 f.write("echo ''\n")
852 f.write(" ".join(patchcmd) + "\n")
853 os.chmod(rcfile, 0o775)
854
855 self.terminal("bash --rcfile " + rcfile, 'Patch Rejects: Please fix patch rejects manually', self.patchset.d)
856
857 # Construct a new PatchSet after the user's changes, compare the
858 # sets, checking patches for modifications, and doing a remote
859 # refresh on each.
860 oldpatchset = self.patchset
861 self.patchset = oldpatchset.__class__(self.patchset.dir, self.patchset.d)
862
863 for patch in self.patchset.patches:
864 oldpatch = None
865 for opatch in oldpatchset.patches:
866 if opatch["quiltfile"] == patch["quiltfile"]:
867 oldpatch = opatch
868
869 if oldpatch:
870 patch["remote"] = oldpatch["remote"]
871 if patch["quiltfile"] == oldpatch["quiltfile"]:
872 if patch["quiltfilemd5"] != oldpatch["quiltfilemd5"]:
873 bb.note("Patch %s has changed, updating remote url %s" % (os.path.basename(patch["quiltfile"]), patch["remote"]))
874 # user change? remote refresh
875 self.patchset.Refresh(remote=True, patch=self.patchset.patches.index(patch))
876 else:
877 # User did not fix the problem. Abort.
878 raise PatchError("Patch application failed, and user did not fix and refresh the patch.")
879 except Exception:
880 raise
881 finally:
882 os.chdir(olddir)
883
884
885def patch_path(url, fetch, workdir, expand=True):
886 """Return the local path of a patch, or return nothing if this isn't a patch"""
887
888 local = fetch.localpath(url)
889 if os.path.isdir(local):
890 return
891 base, ext = os.path.splitext(os.path.basename(local))
892 if ext in ('.gz', '.bz2', '.xz', '.Z'):
893 if expand:
894 local = os.path.join(workdir, base)
895 ext = os.path.splitext(base)[1]
896
897 urldata = fetch.ud[url]
898 if "apply" in urldata.parm:
899 apply = oe.types.boolean(urldata.parm["apply"])
900 if not apply:
901 return
902 elif ext not in (".diff", ".patch"):
903 return
904
905 return local
906
907def src_patches(d, all=False, expand=True):
908 workdir = d.getVar('WORKDIR')
909 fetch = bb.fetch2.Fetch([], d)
910 patches = []
911 sources = []
912 for url in fetch.urls:
913 local = patch_path(url, fetch, workdir, expand)
914 if not local:
915 if all:
916 local = fetch.localpath(url)
917 sources.append(local)
918 continue
919
920 urldata = fetch.ud[url]
921 parm = urldata.parm
922 patchname = parm.get('pname') or os.path.basename(local)
923
924 apply, reason = should_apply(parm, d)
925 if not apply:
926 if reason:
927 bb.note("Patch %s %s" % (patchname, reason))
928 continue
929
930 patchparm = {'patchname': patchname}
931 if "striplevel" in parm:
932 striplevel = parm["striplevel"]
933 elif "pnum" in parm:
934 #bb.msg.warn(None, "Deprecated usage of 'pnum' url parameter in '%s', please use 'striplevel'" % url)
935 striplevel = parm["pnum"]
936 else:
937 striplevel = '1'
938 patchparm['striplevel'] = striplevel
939
940 patchdir = parm.get('patchdir')
941 if patchdir:
942 patchparm['patchdir'] = patchdir
943
944 localurl = bb.fetch.encodeurl(('file', '', local, '', '', patchparm))
945 patches.append(localurl)
946
947 if all:
948 return sources
949
950 return patches
951
952
953def should_apply(parm, d):
954 import bb.utils
955 if "mindate" in parm or "maxdate" in parm:
956 pn = d.getVar('PN')
957 srcdate = d.getVar('SRCDATE_%s' % pn)
958 if not srcdate:
959 srcdate = d.getVar('SRCDATE')
960
961 if srcdate == "now":
962 srcdate = d.getVar('DATE')
963
964 if "maxdate" in parm and parm["maxdate"] < srcdate:
965 return False, 'is outdated'
966
967 if "mindate" in parm and parm["mindate"] > srcdate:
968 return False, 'is predated'
969
970
971 if "minrev" in parm:
972 srcrev = d.getVar('SRCREV')
973 if srcrev and srcrev < parm["minrev"]:
974 return False, 'applies to later revisions'
975
976 if "maxrev" in parm:
977 srcrev = d.getVar('SRCREV')
978 if srcrev and srcrev > parm["maxrev"]:
979 return False, 'applies to earlier revisions'
980
981 if "rev" in parm:
982 srcrev = d.getVar('SRCREV')
983 if srcrev and parm["rev"] not in srcrev:
984 return False, "doesn't apply to revision"
985
986 if "notrev" in parm:
987 srcrev = d.getVar('SRCREV')
988 if srcrev and parm["notrev"] in srcrev:
989 return False, "doesn't apply to revision"
990
991 if "maxver" in parm:
992 pv = d.getVar('PV')
993 if bb.utils.vercmp_string_op(pv, parm["maxver"], ">"):
994 return False, "applies to earlier version"
995
996 if "minver" in parm:
997 pv = d.getVar('PV')
998 if bb.utils.vercmp_string_op(pv, parm["minver"], "<"):
999 return False, "applies to later version"
1000
1001 return True, None
diff --git a/meta-xilinx-core/lib/oe/path.py b/meta-xilinx-core/lib/oe/path.py
new file mode 100644
index 00000000..5d21cdcb
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/path.py
@@ -0,0 +1,349 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import errno
8import glob
9import shutil
10import subprocess
11import os.path
12
13def join(*paths):
14 """Like os.path.join but doesn't treat absolute RHS specially"""
15 return os.path.normpath("/".join(paths))
16
17def relative(src, dest):
18 """ Return a relative path from src to dest.
19
20 >>> relative("/usr/bin", "/tmp/foo/bar")
21 ../../tmp/foo/bar
22
23 >>> relative("/usr/bin", "/usr/lib")
24 ../lib
25
26 >>> relative("/tmp", "/tmp/foo/bar")
27 foo/bar
28 """
29
30 return os.path.relpath(dest, src)
31
32def make_relative_symlink(path):
33 """ Convert an absolute symlink to a relative one """
34 if not os.path.islink(path):
35 return
36 link = os.readlink(path)
37 if not os.path.isabs(link):
38 return
39
40 # find the common ancestor directory
41 ancestor = path
42 depth = 0
43 while ancestor and not link.startswith(ancestor):
44 ancestor = ancestor.rpartition('/')[0]
45 depth += 1
46
47 if not ancestor:
48 print("make_relative_symlink() Error: unable to find the common ancestor of %s and its target" % path)
49 return
50
51 base = link.partition(ancestor)[2].strip('/')
52 while depth > 1:
53 base = "../" + base
54 depth -= 1
55
56 os.remove(path)
57 os.symlink(base, path)
58
59def replace_absolute_symlinks(basedir, d):
60 """
61 Walk basedir looking for absolute symlinks and replacing them with relative ones.
62 The absolute links are assumed to be relative to basedir
63 (compared to make_relative_symlink above which tries to compute common ancestors
64 using pattern matching instead)
65 """
66 for walkroot, dirs, files in os.walk(basedir):
67 for file in files + dirs:
68 path = os.path.join(walkroot, file)
69 if not os.path.islink(path):
70 continue
71 link = os.readlink(path)
72 if not os.path.isabs(link):
73 continue
74 walkdir = os.path.dirname(path.rpartition(basedir)[2])
75 base = os.path.relpath(link, walkdir)
76 bb.debug(2, "Replacing absolute path %s with relative path %s" % (link, base))
77 os.remove(path)
78 os.symlink(base, path)
79
80def format_display(path, metadata):
81 """ Prepare a path for display to the user. """
82 rel = relative(metadata.getVar("TOPDIR"), path)
83 if len(rel) > len(path):
84 return path
85 else:
86 return rel
87
88def copytree(src, dst):
89 # We could use something like shutil.copytree here but it turns out to
90 # to be slow. It takes twice as long copying to an empty directory.
91 # If dst already has contents performance can be 15 time slower
92 # This way we also preserve hardlinks between files in the tree.
93
94 bb.utils.mkdirhier(dst)
95 cmd = "tar --xattrs --xattrs-include='*' -cf - -S -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst)
96 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
97
98def copyhardlinktree(src, dst):
99 """Make a tree of hard links when possible, otherwise copy."""
100 bb.utils.mkdirhier(dst)
101 if os.path.isdir(src) and not len(os.listdir(src)):
102 return
103
104 canhard = False
105 testfile = None
106 for root, dirs, files in os.walk(src):
107 if len(files):
108 testfile = os.path.join(root, files[0])
109 break
110
111 if testfile is not None:
112 try:
113 os.link(testfile, os.path.join(dst, 'testfile'))
114 os.unlink(os.path.join(dst, 'testfile'))
115 canhard = True
116 except Exception as e:
117 bb.debug(2, "Hardlink test failed with " + str(e))
118
119 if (canhard):
120 # Need to copy directories only with tar first since cp will error if two
121 # writers try and create a directory at the same time
122 cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -S -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xhf - -C %s" % (src, src, dst)
123 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
124 source = ''
125 if os.path.isdir(src):
126 if len(glob.glob('%s/.??*' % src)) > 0:
127 source = './.??* '
128 if len(glob.glob('%s/**' % src)) > 0:
129 source += './*'
130 s_dir = src
131 else:
132 source = src
133 s_dir = os.getcwd()
134 cmd = 'cp -afl --preserve=xattr %s %s' % (source, os.path.realpath(dst))
135 subprocess.check_output(cmd, shell=True, cwd=s_dir, stderr=subprocess.STDOUT)
136 else:
137 copytree(src, dst)
138
139def copyhardlink(src, dst):
140 """Make a hard link when possible, otherwise copy."""
141
142 try:
143 os.link(src, dst)
144 except OSError:
145 shutil.copy(src, dst)
146
147def remove(path, recurse=True):
148 """
149 Equivalent to rm -f or rm -rf
150 NOTE: be careful about passing paths that may contain filenames with
151 wildcards in them (as opposed to passing an actual wildcarded path) -
152 since we use glob.glob() to expand the path. Filenames containing
153 square brackets are particularly problematic since the they may not
154 actually expand to match the original filename.
155 """
156 for name in glob.glob(path):
157 try:
158 os.unlink(name)
159 except OSError as exc:
160 if recurse and exc.errno == errno.EISDIR:
161 shutil.rmtree(name)
162 elif exc.errno != errno.ENOENT:
163 raise
164
165def symlink(source, destination, force=False):
166 """Create a symbolic link"""
167 try:
168 if force:
169 remove(destination)
170 os.symlink(source, destination)
171 except OSError as e:
172 if e.errno != errno.EEXIST or os.readlink(destination) != source:
173 raise
174
175def relsymlink(target, name, force=False):
176 symlink(os.path.relpath(target, os.path.dirname(name)), name, force=force)
177
178def find(dir, **walkoptions):
179 """ Given a directory, recurses into that directory,
180 returning all files as absolute paths. """
181
182 for root, dirs, files in os.walk(dir, **walkoptions):
183 for file in files:
184 yield os.path.join(root, file)
185
186
187## realpath() related functions
188def __is_path_below(file, root):
189 return (file + os.path.sep).startswith(root)
190
191def __realpath_rel(start, rel_path, root, loop_cnt, assume_dir):
192 """Calculates real path of symlink 'start' + 'rel_path' below
193 'root'; no part of 'start' below 'root' must contain symlinks. """
194 have_dir = True
195
196 for d in rel_path.split(os.path.sep):
197 if not have_dir and not assume_dir:
198 raise OSError(errno.ENOENT, "no such directory %s" % start)
199
200 if d == os.path.pardir: # '..'
201 if len(start) >= len(root):
202 # do not follow '..' before root
203 start = os.path.dirname(start)
204 else:
205 # emit warning?
206 pass
207 else:
208 (start, have_dir) = __realpath(os.path.join(start, d),
209 root, loop_cnt, assume_dir)
210
211 assert(__is_path_below(start, root))
212
213 return start
214
215def __realpath(file, root, loop_cnt, assume_dir):
216 while os.path.islink(file) and len(file) >= len(root):
217 if loop_cnt == 0:
218 raise OSError(errno.ELOOP, file)
219
220 loop_cnt -= 1
221 target = os.path.normpath(os.readlink(file))
222
223 if not os.path.isabs(target):
224 tdir = os.path.dirname(file)
225 assert(__is_path_below(tdir, root))
226 else:
227 tdir = root
228
229 file = __realpath_rel(tdir, target, root, loop_cnt, assume_dir)
230
231 try:
232 is_dir = os.path.isdir(file)
233 except:
234 is_dir = false
235
236 return (file, is_dir)
237
238def realpath(file, root, use_physdir = True, loop_cnt = 100, assume_dir = False):
239 """ Returns the canonical path of 'file' with assuming a
240 toplevel 'root' directory. When 'use_physdir' is set, all
241 preceding path components of 'file' will be resolved first;
242 this flag should be set unless it is guaranteed that there is
243 no symlink in the path. When 'assume_dir' is not set, missing
244 path components will raise an ENOENT error"""
245
246 root = os.path.normpath(root)
247 file = os.path.normpath(file)
248
249 if not root.endswith(os.path.sep):
250 # letting root end with '/' makes some things easier
251 root = root + os.path.sep
252
253 if not __is_path_below(file, root):
254 raise OSError(errno.EINVAL, "file '%s' is not below root" % file)
255
256 try:
257 if use_physdir:
258 file = __realpath_rel(root, file[(len(root) - 1):], root, loop_cnt, assume_dir)
259 else:
260 file = __realpath(file, root, loop_cnt, assume_dir)[0]
261 except OSError as e:
262 if e.errno == errno.ELOOP:
263 # make ELOOP more readable; without catching it, there will
264 # be printed a backtrace with 100s of OSError exceptions
265 # else
266 raise OSError(errno.ELOOP,
267 "too much recursions while resolving '%s'; loop in '%s'" %
268 (file, e.strerror))
269
270 raise
271
272 return file
273
274def is_path_parent(possible_parent, *paths):
275 """
276 Return True if a path is the parent of another, False otherwise.
277 Multiple paths to test can be specified in which case all
278 specified test paths must be under the parent in order to
279 return True.
280 """
281 def abs_path_trailing(pth):
282 pth_abs = os.path.abspath(pth)
283 if not pth_abs.endswith(os.sep):
284 pth_abs += os.sep
285 return pth_abs
286
287 possible_parent_abs = abs_path_trailing(possible_parent)
288 if not paths:
289 return False
290 for path in paths:
291 path_abs = abs_path_trailing(path)
292 if not path_abs.startswith(possible_parent_abs):
293 return False
294 return True
295
296def which_wild(pathname, path=None, mode=os.F_OK, *, reverse=False, candidates=False):
297 """Search a search path for pathname, supporting wildcards.
298
299 Return all paths in the specific search path matching the wildcard pattern
300 in pathname, returning only the first encountered for each file. If
301 candidates is True, information on all potential candidate paths are
302 included.
303 """
304 paths = (path or os.environ.get('PATH', os.defpath)).split(':')
305 if reverse:
306 paths.reverse()
307
308 seen, files = set(), []
309 for index, element in enumerate(paths):
310 if not os.path.isabs(element):
311 element = os.path.abspath(element)
312
313 candidate = os.path.join(element, pathname)
314 globbed = glob.glob(candidate)
315 if globbed:
316 for found_path in sorted(globbed):
317 if not os.access(found_path, mode):
318 continue
319 rel = os.path.relpath(found_path, element)
320 if rel not in seen:
321 seen.add(rel)
322 if candidates:
323 files.append((found_path, [os.path.join(p, rel) for p in paths[:index+1]]))
324 else:
325 files.append(found_path)
326
327 return files
328
329def canonicalize(paths, sep=','):
330 """Given a string with paths (separated by commas by default), expand
331 each path using os.path.realpath() and return the resulting paths as a
332 string (separated using the same separator a the original string).
333 """
334 # Ignore paths containing "$" as they are assumed to be unexpanded bitbake
335 # variables. Normally they would be ignored, e.g., when passing the paths
336 # through the shell they would expand to empty strings. However, when they
337 # are passed through os.path.realpath(), it will cause them to be prefixed
338 # with the absolute path to the current directory and thus not be empty
339 # anymore.
340 #
341 # Also maintain trailing slashes, as the paths may actually be used as
342 # prefixes in sting compares later on, where the slashes then are important.
343 canonical_paths = []
344 for path in (paths or '').split(sep):
345 if '$' not in path:
346 trailing_slash = path.endswith('/') and '/' or ''
347 canonical_paths.append(os.path.realpath(path) + trailing_slash)
348
349 return sep.join(canonical_paths)
diff --git a/meta-xilinx-core/lib/oe/prservice.py b/meta-xilinx-core/lib/oe/prservice.py
new file mode 100644
index 00000000..c41242c8
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/prservice.py
@@ -0,0 +1,127 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7def prserv_make_conn(d, check = False):
8 import prserv.serv
9 host_params = list([_f for _f in (d.getVar("PRSERV_HOST") or '').split(':') if _f])
10 try:
11 conn = None
12 conn = prserv.serv.connect(host_params[0], int(host_params[1]))
13 if check:
14 if not conn.ping():
15 raise Exception('service not available')
16 except Exception as exc:
17 bb.fatal("Connecting to PR service %s:%s failed: %s" % (host_params[0], host_params[1], str(exc)))
18
19 return conn
20
21def prserv_dump_db(d):
22 if not d.getVar('PRSERV_HOST'):
23 bb.error("Not using network based PR service")
24 return None
25
26 conn = prserv_make_conn(d)
27 if conn is None:
28 bb.error("Making connection failed to remote PR service")
29 return None
30
31 #dump db
32 opt_version = d.getVar('PRSERV_DUMPOPT_VERSION')
33 opt_pkgarch = d.getVar('PRSERV_DUMPOPT_PKGARCH')
34 opt_checksum = d.getVar('PRSERV_DUMPOPT_CHECKSUM')
35 opt_col = ("1" == d.getVar('PRSERV_DUMPOPT_COL'))
36 d = conn.export(opt_version, opt_pkgarch, opt_checksum, opt_col)
37 conn.close()
38 return d
39
40def prserv_import_db(d, filter_version=None, filter_pkgarch=None, filter_checksum=None):
41 if not d.getVar('PRSERV_HOST'):
42 bb.error("Not using network based PR service")
43 return None
44
45 conn = prserv_make_conn(d)
46 if conn is None:
47 bb.error("Making connection failed to remote PR service")
48 return None
49 #get the entry values
50 imported = []
51 prefix = "PRAUTO$"
52 for v in d.keys():
53 if v.startswith(prefix):
54 (remain, sep, checksum) = v.rpartition('$')
55 (remain, sep, pkgarch) = remain.rpartition('$')
56 (remain, sep, version) = remain.rpartition('$')
57 if (remain + '$' != prefix) or \
58 (filter_version and filter_version != version) or \
59 (filter_pkgarch and filter_pkgarch != pkgarch) or \
60 (filter_checksum and filter_checksum != checksum):
61 continue
62 try:
63 value = int(d.getVar(remain + '$' + version + '$' + pkgarch + '$' + checksum))
64 except BaseException as exc:
65 bb.debug("Not valid value of %s:%s" % (v,str(exc)))
66 continue
67 ret = conn.importone(version,pkgarch,checksum,value)
68 if ret != value:
69 bb.error("importing(%s,%s,%s,%d) failed. DB may have larger value %d" % (version,pkgarch,checksum,value,ret))
70 else:
71 imported.append((version,pkgarch,checksum,value))
72 conn.close()
73 return imported
74
75def prserv_export_tofile(d, metainfo, datainfo, lockdown, nomax=False):
76 import bb.utils
77 #initilize the output file
78 bb.utils.mkdirhier(d.getVar('PRSERV_DUMPDIR'))
79 df = d.getVar('PRSERV_DUMPFILE')
80 #write data
81 with open(df, "a") as f, bb.utils.fileslocked(["%s.lock" % df]) as locks:
82 if metainfo:
83 #dump column info
84 f.write("#PR_core_ver = \"%s\"\n\n" % metainfo['core_ver']);
85 f.write("#Table: %s\n" % metainfo['tbl_name'])
86 f.write("#Columns:\n")
87 f.write("#name \t type \t notn \t dflt \t pk\n")
88 f.write("#----------\t --------\t --------\t --------\t ----\n")
89 for i in range(len(metainfo['col_info'])):
90 f.write("#%10s\t %8s\t %8s\t %8s\t %4s\n" %
91 (metainfo['col_info'][i]['name'],
92 metainfo['col_info'][i]['type'],
93 metainfo['col_info'][i]['notnull'],
94 metainfo['col_info'][i]['dflt_value'],
95 metainfo['col_info'][i]['pk']))
96 f.write("\n")
97
98 if lockdown:
99 f.write("PRSERV_LOCKDOWN = \"1\"\n\n")
100
101 if datainfo:
102 idx = {}
103 for i in range(len(datainfo)):
104 pkgarch = datainfo[i]['pkgarch']
105 value = datainfo[i]['value']
106 if pkgarch not in idx:
107 idx[pkgarch] = i
108 elif value > datainfo[idx[pkgarch]]['value']:
109 idx[pkgarch] = i
110 f.write("PRAUTO$%s$%s$%s = \"%s\"\n" %
111 (str(datainfo[i]['version']), pkgarch, str(datainfo[i]['checksum']), str(value)))
112 if not nomax:
113 for i in idx:
114 f.write("PRAUTO_%s_%s = \"%s\"\n" % (str(datainfo[idx[i]]['version']),str(datainfo[idx[i]]['pkgarch']),str(datainfo[idx[i]]['value'])))
115
116def prserv_check_avail(d):
117 host_params = list([_f for _f in (d.getVar("PRSERV_HOST") or '').split(':') if _f])
118 try:
119 if len(host_params) != 2:
120 raise TypeError
121 else:
122 int(host_params[1])
123 except TypeError:
124 bb.fatal('Undefined/incorrect PRSERV_HOST value. Format: "host:port"')
125 else:
126 conn = prserv_make_conn(d, True)
127 conn.close()
diff --git a/meta-xilinx-core/lib/oe/qa.py b/meta-xilinx-core/lib/oe/qa.py
new file mode 100644
index 00000000..f8ae3c74
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/qa.py
@@ -0,0 +1,238 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import os, struct, mmap
8
9class NotELFFileError(Exception):
10 pass
11
12class ELFFile:
13 EI_NIDENT = 16
14
15 EI_CLASS = 4
16 EI_DATA = 5
17 EI_VERSION = 6
18 EI_OSABI = 7
19 EI_ABIVERSION = 8
20
21 E_MACHINE = 0x12
22
23 # possible values for EI_CLASS
24 ELFCLASSNONE = 0
25 ELFCLASS32 = 1
26 ELFCLASS64 = 2
27
28 # possible value for EI_VERSION
29 EV_CURRENT = 1
30
31 # possible values for EI_DATA
32 EI_DATA_NONE = 0
33 EI_DATA_LSB = 1
34 EI_DATA_MSB = 2
35
36 PT_INTERP = 3
37
38 def my_assert(self, expectation, result):
39 if not expectation == result:
40 #print "'%x','%x' %s" % (ord(expectation), ord(result), self.name)
41 raise NotELFFileError("%s is not an ELF" % self.name)
42
43 def __init__(self, name):
44 self.name = name
45 self.objdump_output = {}
46 self.data = None
47
48 # Context Manager functions to close the mmap explicitly
49 def __enter__(self):
50 return self
51
52 def __exit__(self, exc_type, exc_value, traceback):
53 self.close()
54
55 def close(self):
56 if self.data:
57 self.data.close()
58
59 def open(self):
60 with open(self.name, "rb") as f:
61 try:
62 self.data = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
63 except ValueError:
64 # This means the file is empty
65 raise NotELFFileError("%s is empty" % self.name)
66
67 # Check the file has the minimum number of ELF table entries
68 if len(self.data) < ELFFile.EI_NIDENT + 4:
69 raise NotELFFileError("%s is not an ELF" % self.name)
70
71 # ELF header
72 self.my_assert(self.data[0], 0x7f)
73 self.my_assert(self.data[1], ord('E'))
74 self.my_assert(self.data[2], ord('L'))
75 self.my_assert(self.data[3], ord('F'))
76 if self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS32:
77 self.bits = 32
78 elif self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS64:
79 self.bits = 64
80 else:
81 # Not 32-bit or 64.. lets assert
82 raise NotELFFileError("ELF but not 32 or 64 bit.")
83 self.my_assert(self.data[ELFFile.EI_VERSION], ELFFile.EV_CURRENT)
84
85 self.endian = self.data[ELFFile.EI_DATA]
86 if self.endian not in (ELFFile.EI_DATA_LSB, ELFFile.EI_DATA_MSB):
87 raise NotELFFileError("Unexpected EI_DATA %x" % self.endian)
88
89 def osAbi(self):
90 return self.data[ELFFile.EI_OSABI]
91
92 def abiVersion(self):
93 return self.data[ELFFile.EI_ABIVERSION]
94
95 def abiSize(self):
96 return self.bits
97
98 def isLittleEndian(self):
99 return self.endian == ELFFile.EI_DATA_LSB
100
101 def isBigEndian(self):
102 return self.endian == ELFFile.EI_DATA_MSB
103
104 def getStructEndian(self):
105 return {ELFFile.EI_DATA_LSB: "<",
106 ELFFile.EI_DATA_MSB: ">"}[self.endian]
107
108 def getShort(self, offset):
109 return struct.unpack_from(self.getStructEndian() + "H", self.data, offset)[0]
110
111 def getWord(self, offset):
112 return struct.unpack_from(self.getStructEndian() + "i", self.data, offset)[0]
113
114 def isDynamic(self):
115 """
116 Return True if there is a .interp segment (therefore dynamically
117 linked), otherwise False (statically linked).
118 """
119 offset = self.getWord(self.bits == 32 and 0x1C or 0x20)
120 size = self.getShort(self.bits == 32 and 0x2A or 0x36)
121 count = self.getShort(self.bits == 32 and 0x2C or 0x38)
122
123 for i in range(0, count):
124 p_type = self.getWord(offset + i * size)
125 if p_type == ELFFile.PT_INTERP:
126 return True
127 return False
128
129 def machine(self):
130 """
131 We know the endian stored in self.endian and we
132 know the position
133 """
134 return self.getShort(ELFFile.E_MACHINE)
135
136 def set_objdump(self, cmd, output):
137 self.objdump_output[cmd] = output
138
139 def run_objdump(self, cmd, d):
140 import bb.process
141 import sys
142
143 if cmd in self.objdump_output:
144 return self.objdump_output[cmd]
145
146 objdump = d.getVar('OBJDUMP')
147
148 env = os.environ.copy()
149 env["LC_ALL"] = "C"
150 env["PATH"] = d.getVar('PATH')
151
152 try:
153 bb.note("%s %s %s" % (objdump, cmd, self.name))
154 self.objdump_output[cmd] = bb.process.run([objdump, cmd, self.name], env=env, shell=False)[0]
155 return self.objdump_output[cmd]
156 except Exception as e:
157 bb.note("%s %s %s failed: %s" % (objdump, cmd, self.name, e))
158 return ""
159
160def elf_machine_to_string(machine):
161 """
162 Return the name of a given ELF e_machine field or the hex value as a string
163 if it isn't recognised.
164 """
165 try:
166 return {
167 0x00: "Unset",
168 0x02: "SPARC",
169 0x03: "x86",
170 0x08: "MIPS",
171 0x14: "PowerPC",
172 0x28: "ARM",
173 0x2A: "SuperH",
174 0x32: "IA-64",
175 0x3E: "x86-64",
176 0xB7: "AArch64",
177 0xF7: "BPF"
178 }[machine]
179 except:
180 return "Unknown (%s)" % repr(machine)
181
182def write_error(type, error, d):
183 logfile = d.getVar('QA_LOGFILE')
184 if logfile:
185 p = d.getVar('P')
186 with open(logfile, "a+") as f:
187 f.write("%s: %s [%s]\n" % (p, error, type))
188
189def handle_error(error_class, error_msg, d):
190 if error_class in (d.getVar("ERROR_QA") or "").split():
191 write_error(error_class, error_msg, d)
192 bb.error("QA Issue: %s [%s]" % (error_msg, error_class))
193 d.setVar("QA_ERRORS_FOUND", "True")
194 return False
195 elif error_class in (d.getVar("WARN_QA") or "").split():
196 write_error(error_class, error_msg, d)
197 bb.warn("QA Issue: %s [%s]" % (error_msg, error_class))
198 else:
199 bb.note("QA Issue: %s [%s]" % (error_msg, error_class))
200 return True
201
202def add_message(messages, section, new_msg):
203 if section not in messages:
204 messages[section] = new_msg
205 else:
206 messages[section] = messages[section] + "\n" + new_msg
207
208def exit_with_message_if_errors(message, d):
209 qa_fatal_errors = bb.utils.to_boolean(d.getVar("QA_ERRORS_FOUND"), False)
210 if qa_fatal_errors:
211 bb.fatal(message)
212
213def exit_if_errors(d):
214 exit_with_message_if_errors("Fatal QA errors were found, failing task.", d)
215
216def check_upstream_status(fullpath):
217 import re
218 kinda_status_re = re.compile(r"^.*upstream.*status.*$", re.IGNORECASE | re.MULTILINE)
219 strict_status_re = re.compile(r"^Upstream-Status: (Pending|Submitted|Denied|Inappropriate|Backport|Inactive-Upstream)( .+)?$", re.MULTILINE)
220 guidelines = "https://docs.yoctoproject.org/contributor-guide/recipe-style-guide.html#patch-upstream-status"
221
222 with open(fullpath, encoding='utf-8', errors='ignore') as f:
223 file_content = f.read()
224 match_kinda = kinda_status_re.search(file_content)
225 match_strict = strict_status_re.search(file_content)
226
227 if not match_strict:
228 if match_kinda:
229 return "Malformed Upstream-Status in patch\n%s\nPlease correct according to %s :\n%s" % (fullpath, guidelines, match_kinda.group(0))
230 else:
231 return "Missing Upstream-Status in patch\n%s\nPlease add according to %s ." % (fullpath, guidelines)
232
233if __name__ == "__main__":
234 import sys
235
236 with ELFFile(sys.argv[1]) as elf:
237 elf.open()
238 print(elf.isDynamic())
diff --git a/meta-xilinx-core/lib/oe/recipeutils.py b/meta-xilinx-core/lib/oe/recipeutils.py
new file mode 100644
index 00000000..de1fbdd3
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/recipeutils.py
@@ -0,0 +1,1185 @@
1# Utility functions for reading and modifying recipes
2#
3# Some code borrowed from the OE layer index
4#
5# Copyright (C) 2013-2017 Intel Corporation
6#
7# SPDX-License-Identifier: GPL-2.0-only
8#
9
10import sys
11import os
12import os.path
13import tempfile
14import textwrap
15import difflib
16from . import utils
17import shutil
18import re
19import fnmatch
20import glob
21import bb.tinfoil
22
23from collections import OrderedDict, defaultdict
24from bb.utils import vercmp_string
25
26# Help us to find places to insert values
27recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()', 'BBCLASSEXTEND']
28# Variables that sometimes are a bit long but shouldn't be wrapped
29nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', r'SRC_URI\[(.+\.)?md5sum\]', r'SRC_URI\[(.+\.)?sha[0-9]+sum\]']
30list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM']
31meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
32
33
34def simplify_history(history, d):
35 """
36 Eliminate any irrelevant events from a variable history
37 """
38 ret_history = []
39 has_set = False
40 # Go backwards through the history and remove any immediate operations
41 # before the most recent set
42 for event in reversed(history):
43 if 'flag' in event or not 'file' in event:
44 continue
45 if event['op'] == 'set':
46 if has_set:
47 continue
48 has_set = True
49 elif event['op'] in ('append', 'prepend', 'postdot', 'predot'):
50 # Reminder: "append" and "prepend" mean += and =+ respectively, NOT :append / :prepend
51 if has_set:
52 continue
53 ret_history.insert(0, event)
54 return ret_history
55
56
57def get_var_files(fn, varlist, d):
58 """Find the file in which each of a list of variables is set.
59 Note: requires variable history to be enabled when parsing.
60 """
61 varfiles = {}
62 for v in varlist:
63 files = []
64 if '[' in v:
65 varsplit = v.split('[')
66 varflag = varsplit[1].split(']')[0]
67 history = d.varhistory.variable(varsplit[0])
68 for event in history:
69 if 'file' in event and event.get('flag', '') == varflag:
70 files.append(event['file'])
71 else:
72 history = d.varhistory.variable(v)
73 for event in history:
74 if 'file' in event and not 'flag' in event:
75 files.append(event['file'])
76 if files:
77 actualfile = files[-1]
78 else:
79 actualfile = None
80 varfiles[v] = actualfile
81
82 return varfiles
83
84
85def split_var_value(value, assignment=True):
86 """
87 Split a space-separated variable's value into a list of items,
88 taking into account that some of the items might be made up of
89 expressions containing spaces that should not be split.
90 Parameters:
91 value:
92 The string value to split
93 assignment:
94 True to assume that the value represents an assignment
95 statement, False otherwise. If True, and an assignment
96 statement is passed in the first item in
97 the returned list will be the part of the assignment
98 statement up to and including the opening quote character,
99 and the last item will be the closing quote.
100 """
101 inexpr = 0
102 lastchar = None
103 out = []
104 buf = ''
105 for char in value:
106 if char == '{':
107 if lastchar == '$':
108 inexpr += 1
109 elif char == '}':
110 inexpr -= 1
111 elif assignment and char in '"\'' and inexpr == 0:
112 if buf:
113 out.append(buf)
114 out.append(char)
115 char = ''
116 buf = ''
117 elif char.isspace() and inexpr == 0:
118 char = ''
119 if buf:
120 out.append(buf)
121 buf = ''
122 buf += char
123 lastchar = char
124 if buf:
125 out.append(buf)
126
127 # Join together assignment statement and opening quote
128 outlist = out
129 if assignment:
130 assigfound = False
131 for idx, item in enumerate(out):
132 if '=' in item:
133 assigfound = True
134 if assigfound:
135 if '"' in item or "'" in item:
136 outlist = [' '.join(out[:idx+1])]
137 outlist.extend(out[idx+1:])
138 break
139 return outlist
140
141
142def patch_recipe_lines(fromlines, values, trailing_newline=True):
143 """Update or insert variable values into lines from a recipe.
144 Note that some manual inspection/intervention may be required
145 since this cannot handle all situations.
146 """
147
148 import bb.utils
149
150 if trailing_newline:
151 newline = '\n'
152 else:
153 newline = ''
154
155 nowrap_vars_res = []
156 for item in nowrap_vars:
157 nowrap_vars_res.append(re.compile('^%s$' % item))
158
159 recipe_progression_res = []
160 recipe_progression_restrs = []
161 for item in recipe_progression:
162 if item.endswith('()'):
163 key = item[:-2]
164 else:
165 key = item
166 restr = r'%s(_[a-zA-Z0-9-_$(){}]+|\[[^\]]*\])?' % key
167 if item.endswith('()'):
168 recipe_progression_restrs.append(restr + '()')
169 else:
170 recipe_progression_restrs.append(restr)
171 recipe_progression_res.append(re.compile('^%s$' % restr))
172
173 def get_recipe_pos(variable):
174 for i, p in enumerate(recipe_progression_res):
175 if p.match(variable):
176 return i
177 return -1
178
179 remainingnames = {}
180 for k in values.keys():
181 remainingnames[k] = get_recipe_pos(k)
182 remainingnames = OrderedDict(sorted(remainingnames.items(), key=lambda x: x[1]))
183
184 modifying = False
185
186 def outputvalue(name, lines, rewindcomments=False):
187 if values[name] is None:
188 return
189 if isinstance(values[name], tuple):
190 op, value = values[name]
191 if op == '+=' and value.strip() == '':
192 return
193 else:
194 value = values[name]
195 op = '='
196 rawtext = '%s %s "%s"%s' % (name, op, value, newline)
197 addlines = []
198 nowrap = False
199 for nowrap_re in nowrap_vars_res:
200 if nowrap_re.match(name):
201 nowrap = True
202 break
203 if nowrap:
204 addlines.append(rawtext)
205 elif name in list_vars:
206 splitvalue = split_var_value(value, assignment=False)
207 if len(splitvalue) > 1:
208 linesplit = ' \\\n' + (' ' * (len(name) + 4))
209 addlines.append('%s %s "%s%s"%s' % (name, op, linesplit.join(splitvalue), linesplit, newline))
210 else:
211 addlines.append(rawtext)
212 else:
213 wrapped = textwrap.wrap(rawtext)
214 for wrapline in wrapped[:-1]:
215 addlines.append('%s \\%s' % (wrapline, newline))
216 addlines.append('%s%s' % (wrapped[-1], newline))
217
218 # Split on newlines - this isn't strictly necessary if you are only
219 # going to write the output to disk, but if you want to compare it
220 # (as patch_recipe_file() will do if patch=True) then it's important.
221 addlines = [line for l in addlines for line in l.splitlines(True)]
222 if rewindcomments:
223 # Ensure we insert the lines before any leading comments
224 # (that we'd want to ensure remain leading the next value)
225 for i, ln in reversed(list(enumerate(lines))):
226 if not ln.startswith('#'):
227 lines[i+1:i+1] = addlines
228 break
229 else:
230 lines.extend(addlines)
231 else:
232 lines.extend(addlines)
233
234 existingnames = []
235 def patch_recipe_varfunc(varname, origvalue, op, newlines):
236 if modifying:
237 # Insert anything that should come before this variable
238 pos = get_recipe_pos(varname)
239 for k in list(remainingnames):
240 if remainingnames[k] > -1 and pos >= remainingnames[k] and not k in existingnames:
241 outputvalue(k, newlines, rewindcomments=True)
242 del remainingnames[k]
243 # Now change this variable, if it needs to be changed
244 if varname in existingnames and op in ['+=', '=', '=+']:
245 if varname in remainingnames:
246 outputvalue(varname, newlines)
247 del remainingnames[varname]
248 return None, None, 0, True
249 else:
250 if varname in values:
251 existingnames.append(varname)
252 return origvalue, None, 0, True
253
254 # First run - establish which values we want to set are already in the file
255 varlist = [re.escape(item) for item in values.keys()]
256 bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc)
257 # Second run - actually set everything
258 modifying = True
259 varlist.extend(recipe_progression_restrs)
260 changed, tolines = bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc, match_overrides=True)
261
262 if remainingnames:
263 if tolines and tolines[-1].strip() != '':
264 tolines.append('\n')
265 for k in remainingnames.keys():
266 outputvalue(k, tolines)
267
268 return changed, tolines
269
270
271def patch_recipe_file(fn, values, patch=False, relpath='', redirect_output=None):
272 """Update or insert variable values into a recipe file (assuming you
273 have already identified the exact file you want to update.)
274 Note that some manual inspection/intervention may be required
275 since this cannot handle all situations.
276 """
277
278 with open(fn, 'r') as f:
279 fromlines = f.readlines()
280
281 _, tolines = patch_recipe_lines(fromlines, values)
282
283 if redirect_output:
284 with open(os.path.join(redirect_output, os.path.basename(fn)), 'w') as f:
285 f.writelines(tolines)
286 return None
287 elif patch:
288 relfn = os.path.relpath(fn, relpath)
289 diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
290 return diff
291 else:
292 with open(fn, 'w') as f:
293 f.writelines(tolines)
294 return None
295
296
297def localise_file_vars(fn, varfiles, varlist):
298 """Given a list of variables and variable history (fetched with get_var_files())
299 find where each variable should be set/changed. This handles for example where a
300 recipe includes an inc file where variables might be changed - in most cases
301 we want to update the inc file when changing the variable value rather than adding
302 it to the recipe itself.
303 """
304 fndir = os.path.dirname(fn) + os.sep
305
306 first_meta_file = None
307 for v in meta_vars:
308 f = varfiles.get(v, None)
309 if f:
310 actualdir = os.path.dirname(f) + os.sep
311 if actualdir.startswith(fndir):
312 first_meta_file = f
313 break
314
315 filevars = defaultdict(list)
316 for v in varlist:
317 f = varfiles[v]
318 # Only return files that are in the same directory as the recipe or in some directory below there
319 # (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable
320 # in if we were going to set a value specific to this recipe)
321 if f:
322 actualfile = f
323 else:
324 # Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it
325 if first_meta_file:
326 actualfile = first_meta_file
327 else:
328 actualfile = fn
329
330 actualdir = os.path.dirname(actualfile) + os.sep
331 if not actualdir.startswith(fndir):
332 actualfile = fn
333 filevars[actualfile].append(v)
334
335 return filevars
336
337def patch_recipe(d, fn, varvalues, patch=False, relpath='', redirect_output=None):
338 """Modify a list of variable values in the specified recipe. Handles inc files if
339 used by the recipe.
340 """
341 overrides = d.getVar('OVERRIDES').split(':')
342 def override_applicable(hevent):
343 op = hevent['op']
344 if '[' in op:
345 opoverrides = op.split('[')[1].split(']')[0].split(':')
346 for opoverride in opoverrides:
347 if not opoverride in overrides:
348 return False
349 return True
350
351 varlist = varvalues.keys()
352 fn = os.path.abspath(fn)
353 varfiles = get_var_files(fn, varlist, d)
354 locs = localise_file_vars(fn, varfiles, varlist)
355 patches = []
356 for f,v in locs.items():
357 vals = {k: varvalues[k] for k in v}
358 f = os.path.abspath(f)
359 if f == fn:
360 extravals = {}
361 for var, value in vals.items():
362 if var in list_vars:
363 history = simplify_history(d.varhistory.variable(var), d)
364 recipe_set = False
365 for event in history:
366 if os.path.abspath(event['file']) == fn:
367 if event['op'] == 'set':
368 recipe_set = True
369 if not recipe_set:
370 for event in history:
371 if event['op'].startswith(':remove'):
372 continue
373 if not override_applicable(event):
374 continue
375 newvalue = value.replace(event['detail'], '')
376 if newvalue == value and os.path.abspath(event['file']) == fn and event['op'].startswith(':'):
377 op = event['op'].replace('[', ':').replace(']', '')
378 extravals[var + op] = None
379 value = newvalue
380 vals[var] = ('+=', value)
381 vals.update(extravals)
382 patchdata = patch_recipe_file(f, vals, patch, relpath, redirect_output)
383 if patch:
384 patches.append(patchdata)
385
386 if patch:
387 return patches
388 else:
389 return None
390
391
392
393def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True, all_variants=False):
394 """Copy (local) recipe files, including both files included via include/require,
395 and files referred to in the SRC_URI variable."""
396 import bb.fetch2
397 import oe.path
398
399 # FIXME need a warning if the unexpanded SRC_URI value contains variable references
400
401 uri_values = []
402 localpaths = []
403 def fetch_urls(rdata):
404 # Collect the local paths from SRC_URI
405 srcuri = rdata.getVar('SRC_URI') or ""
406 if srcuri not in uri_values:
407 fetch = bb.fetch2.Fetch(srcuri.split(), rdata)
408 if download:
409 fetch.download()
410 for pth in fetch.localpaths():
411 if pth not in localpaths:
412 localpaths.append(os.path.abspath(pth))
413 uri_values.append(srcuri)
414
415 fetch_urls(d)
416 if all_variants:
417 # Get files for other variants e.g. in the case of a SRC_URI:append
418 localdata = bb.data.createCopy(d)
419 variants = (localdata.getVar('BBCLASSEXTEND') or '').split()
420 if variants:
421 # Ensure we handle class-target if we're dealing with one of the variants
422 variants.append('target')
423 for variant in variants:
424 localdata.setVar('CLASSOVERRIDE', 'class-%s' % variant)
425 fetch_urls(localdata)
426
427 # Copy local files to target directory and gather any remote files
428 bb_dir = os.path.abspath(os.path.dirname(d.getVar('FILE'))) + os.sep
429 remotes = []
430 copied = []
431 # Need to do this in two steps since we want to check against the absolute path
432 includes = [os.path.abspath(path) for path in d.getVar('BBINCLUDED').split() if os.path.exists(path)]
433 # We also check this below, but we don't want any items in this list being considered remotes
434 includes = [path for path in includes if path.startswith(bb_dir)]
435 for path in localpaths + includes:
436 # Only import files that are under the meta directory
437 if path.startswith(bb_dir):
438 if not whole_dir:
439 relpath = os.path.relpath(path, bb_dir)
440 subdir = os.path.join(tgt_dir, os.path.dirname(relpath))
441 if not os.path.exists(subdir):
442 os.makedirs(subdir)
443 shutil.copy2(path, os.path.join(tgt_dir, relpath))
444 copied.append(relpath)
445 else:
446 remotes.append(path)
447 # Simply copy whole meta dir, if requested
448 if whole_dir:
449 shutil.copytree(bb_dir, tgt_dir)
450
451 return copied, remotes
452
453
454def get_recipe_local_files(d, patches=False, archives=False):
455 """Get a list of local files in SRC_URI within a recipe."""
456 import oe.patch
457 uris = (d.getVar('SRC_URI') or "").split()
458 fetch = bb.fetch2.Fetch(uris, d)
459 # FIXME this list should be factored out somewhere else (such as the
460 # fetcher) though note that this only encompasses actual container formats
461 # i.e. that can contain multiple files as opposed to those that only
462 # contain a compressed stream (i.e. .tar.gz as opposed to just .gz)
463 archive_exts = ['.tar', '.tgz', '.tar.gz', '.tar.Z', '.tbz', '.tbz2', '.tar.bz2', '.txz', '.tar.xz', '.tar.lz', '.zip', '.jar', '.rpm', '.srpm', '.deb', '.ipk', '.tar.7z', '.7z']
464 ret = {}
465 for uri in uris:
466 if fetch.ud[uri].type == 'file':
467 if (not patches and
468 oe.patch.patch_path(uri, fetch, '', expand=False)):
469 continue
470 # Skip files that are referenced by absolute path
471 fname = fetch.ud[uri].basepath
472 if os.path.isabs(fname):
473 continue
474 # Handle subdir=
475 subdir = fetch.ud[uri].parm.get('subdir', '')
476 if subdir:
477 if os.path.isabs(subdir):
478 continue
479 fname = os.path.join(subdir, fname)
480 localpath = fetch.localpath(uri)
481 if not archives:
482 # Ignore archives that will be unpacked
483 if localpath.endswith(tuple(archive_exts)):
484 unpack = fetch.ud[uri].parm.get('unpack', True)
485 if unpack:
486 continue
487 if os.path.isdir(localpath):
488 for root, dirs, files in os.walk(localpath):
489 for fname in files:
490 fileabspath = os.path.join(root,fname)
491 srcdir = os.path.dirname(localpath)
492 ret[os.path.relpath(fileabspath,srcdir)] = fileabspath
493 else:
494 ret[fname] = localpath
495 return ret
496
497
498def get_recipe_patches(d):
499 """Get a list of the patches included in SRC_URI within a recipe."""
500 import oe.patch
501 patches = oe.patch.src_patches(d, expand=False)
502 patchfiles = []
503 for patch in patches:
504 _, _, local, _, _, parm = bb.fetch.decodeurl(patch)
505 patchfiles.append(local)
506 return patchfiles
507
508
509def get_recipe_patched_files(d):
510 """
511 Get the list of patches for a recipe along with the files each patch modifies.
512 Params:
513 d: the datastore for the recipe
514 Returns:
515 a dict mapping patch file path to a list of tuples of changed files and
516 change mode ('A' for add, 'D' for delete or 'M' for modify)
517 """
518 import oe.patch
519 patches = oe.patch.src_patches(d, expand=False)
520 patchedfiles = {}
521 for patch in patches:
522 _, _, patchfile, _, _, parm = bb.fetch.decodeurl(patch)
523 striplevel = int(parm['striplevel'])
524 patchedfiles[patchfile] = oe.patch.PatchSet.getPatchedFiles(patchfile, striplevel, os.path.join(d.getVar('S'), parm.get('patchdir', '')))
525 return patchedfiles
526
527
528def validate_pn(pn):
529 """Perform validation on a recipe name (PN) for a new recipe."""
530 reserved_names = ['forcevariable', 'append', 'prepend', 'remove']
531 if not re.match('^[0-9a-z-.+]+$', pn):
532 return 'Recipe name "%s" is invalid: only characters 0-9, a-z, -, + and . are allowed' % pn
533 elif pn in reserved_names:
534 return 'Recipe name "%s" is invalid: is a reserved keyword' % pn
535 elif pn.startswith('pn-'):
536 return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn
537 elif pn.endswith(('.bb', '.bbappend', '.bbclass', '.inc', '.conf')):
538 return 'Recipe name "%s" is invalid: should be just a name, not a file name' % pn
539 return ''
540
541
542def get_bbfile_path(d, destdir, extrapathhint=None):
543 """
544 Determine the correct path for a recipe within a layer
545 Parameters:
546 d: Recipe-specific datastore
547 destdir: destination directory. Can be the path to the base of the layer or a
548 partial path somewhere within the layer.
549 extrapathhint: a path relative to the base of the layer to try
550 """
551 import bb.cookerdata
552
553 destdir = os.path.abspath(destdir)
554 destlayerdir = find_layerdir(destdir)
555
556 # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf
557 confdata = d.createCopy()
558 confdata.setVar('BBFILES', '')
559 confdata.setVar('LAYERDIR', destlayerdir)
560 destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf")
561 confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata)
562 pn = d.getVar('PN')
563
564 # Parse BBFILES_DYNAMIC and append to BBFILES
565 bbfiles_dynamic = (confdata.getVar('BBFILES_DYNAMIC') or "").split()
566 collections = (confdata.getVar('BBFILE_COLLECTIONS') or "").split()
567 invalid = []
568 for entry in bbfiles_dynamic:
569 parts = entry.split(":", 1)
570 if len(parts) != 2:
571 invalid.append(entry)
572 continue
573 l, f = parts
574 invert = l[0] == "!"
575 if invert:
576 l = l[1:]
577 if (l in collections and not invert) or (l not in collections and invert):
578 confdata.appendVar("BBFILES", " " + f)
579 if invalid:
580 return None
581 bbfilespecs = (confdata.getVar('BBFILES') or '').split()
582 if destdir == destlayerdir:
583 for bbfilespec in bbfilespecs:
584 if not bbfilespec.endswith('.bbappend'):
585 for match in glob.glob(bbfilespec):
586 splitext = os.path.splitext(os.path.basename(match))
587 if splitext[1] == '.bb':
588 mpn = splitext[0].split('_')[0]
589 if mpn == pn:
590 return os.path.dirname(match)
591
592 # Try to make up a path that matches BBFILES
593 # this is a little crude, but better than nothing
594 bpn = d.getVar('BPN')
595 recipefn = os.path.basename(d.getVar('FILE'))
596 pathoptions = [destdir]
597 if extrapathhint:
598 pathoptions.append(os.path.join(destdir, extrapathhint))
599 if destdir == destlayerdir:
600 pathoptions.append(os.path.join(destdir, 'recipes-%s' % bpn, bpn))
601 pathoptions.append(os.path.join(destdir, 'recipes', bpn))
602 pathoptions.append(os.path.join(destdir, bpn))
603 elif not destdir.endswith(('/' + pn, '/' + bpn)):
604 pathoptions.append(os.path.join(destdir, bpn))
605 closepath = ''
606 for pathoption in pathoptions:
607 bbfilepath = os.path.join(pathoption, 'test.bb')
608 for bbfilespec in bbfilespecs:
609 if fnmatch.fnmatchcase(bbfilepath, bbfilespec):
610 return pathoption
611 return None
612
613def get_bbappend_path(d, destlayerdir, wildcardver=False):
614 """Determine how a bbappend for a recipe should be named and located within another layer"""
615
616 import bb.cookerdata
617
618 destlayerdir = os.path.abspath(destlayerdir)
619 recipefile = d.getVar('FILE')
620 recipefn = os.path.splitext(os.path.basename(recipefile))[0]
621 if wildcardver and '_' in recipefn:
622 recipefn = recipefn.split('_', 1)[0] + '_%'
623 appendfn = recipefn + '.bbappend'
624
625 # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf
626 confdata = d.createCopy()
627 confdata.setVar('BBFILES', '')
628 confdata.setVar('LAYERDIR', destlayerdir)
629 destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf")
630 confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata)
631
632 origlayerdir = find_layerdir(recipefile)
633 if not origlayerdir:
634 return (None, False)
635 # Now join this to the path where the bbappend is going and check if it is covered by BBFILES
636 appendpath = os.path.join(destlayerdir, os.path.relpath(os.path.dirname(recipefile), origlayerdir), appendfn)
637 closepath = ''
638 pathok = True
639 for bbfilespec in confdata.getVar('BBFILES').split():
640 if fnmatch.fnmatchcase(appendpath, bbfilespec):
641 # Our append path works, we're done
642 break
643 elif bbfilespec.startswith(destlayerdir) and fnmatch.fnmatchcase('test.bbappend', os.path.basename(bbfilespec)):
644 # Try to find the longest matching path
645 if len(bbfilespec) > len(closepath):
646 closepath = bbfilespec
647 else:
648 # Unfortunately the bbappend layer and the original recipe's layer don't have the same structure
649 if closepath:
650 # bbappend layer's layer.conf at least has a spec that picks up .bbappend files
651 # Now we just need to substitute out any wildcards
652 appendsubdir = os.path.relpath(os.path.dirname(closepath), destlayerdir)
653 if 'recipes-*' in appendsubdir:
654 # Try to copy this part from the original recipe path
655 res = re.search('/recipes-[^/]+/', recipefile)
656 if res:
657 appendsubdir = appendsubdir.replace('/recipes-*/', res.group(0))
658 # This is crude, but we have to do something
659 appendsubdir = appendsubdir.replace('*', recipefn.split('_')[0])
660 appendsubdir = appendsubdir.replace('?', 'a')
661 appendpath = os.path.join(destlayerdir, appendsubdir, appendfn)
662 else:
663 pathok = False
664 return (appendpath, pathok)
665
666
667def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None, params=None, update_original_recipe=False):
668 """
669 Writes a bbappend file for a recipe
670 Parameters:
671 rd: data dictionary for the recipe
672 destlayerdir: base directory of the layer to place the bbappend in
673 (subdirectory path from there will be determined automatically)
674 srcfiles: dict of source files to add to SRC_URI, where the key
675 is the full path to the file to be added, and the value is a
676 dict with following optional keys:
677 path: the original filename as it would appear in SRC_URI
678 or None if it isn't already present.
679 patchdir: the patchdir parameter
680 newname: the name to give to the new added file. None to use
681 the default value: basename(path)
682 You may pass None for this parameter if you simply want to specify
683 your own content via the extralines parameter.
684 install: dict mapping entries in srcfiles to a tuple of two elements:
685 install path (*without* ${D} prefix) and permission value (as a
686 string, e.g. '0644').
687 wildcardver: True to use a % wildcard in the bbappend filename, or
688 False to make the bbappend specific to the recipe version.
689 machine:
690 If specified, make the changes in the bbappend specific to this
691 machine. This will also cause PACKAGE_ARCH = "${MACHINE_ARCH}"
692 to be added to the bbappend.
693 extralines:
694 Extra lines to add to the bbappend. This may be a dict of name
695 value pairs, or simply a list of the lines.
696 removevalues:
697 Variable values to remove - a dict of names/values.
698 redirect_output:
699 If specified, redirects writing the output file to the
700 specified directory (for dry-run purposes)
701 params:
702 Parameters to use when adding entries to SRC_URI. If specified,
703 should be a list of dicts with the same length as srcfiles.
704 update_original_recipe:
705 Force to update the original recipe instead of creating/updating
706 a bbapend. destlayerdir must contain the original recipe
707 """
708
709 if not removevalues:
710 removevalues = {}
711
712 recipefile = rd.getVar('FILE')
713 if update_original_recipe:
714 if destlayerdir not in recipefile:
715 bb.error("destlayerdir %s doesn't contain the original recipe (%s), cannot update it" % (destlayerdir, recipefile))
716 return (None, None)
717
718 appendpath = recipefile
719 else:
720 # Determine how the bbappend should be named
721 appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver)
722 if not appendpath:
723 bb.error('Unable to determine layer directory containing %s' % recipefile)
724 return (None, None)
725 if not pathok:
726 bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath)))
727
728 appenddir = os.path.dirname(appendpath)
729 if not redirect_output:
730 bb.utils.mkdirhier(appenddir)
731
732 # FIXME check if the bbappend doesn't get overridden by a higher priority layer?
733
734 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()]
735 if not os.path.abspath(destlayerdir) in layerdirs:
736 bb.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active')
737
738 bbappendlines = []
739 if extralines:
740 if isinstance(extralines, dict):
741 for name, value in extralines.items():
742 bbappendlines.append((name, '=', value))
743 else:
744 # Do our best to split it
745 for line in extralines:
746 if line[-1] == '\n':
747 line = line[:-1]
748 splitline = line.split(None, 2)
749 if len(splitline) == 3:
750 bbappendlines.append(tuple(splitline))
751 else:
752 raise Exception('Invalid extralines value passed')
753
754 def popline(varname):
755 for i in range(0, len(bbappendlines)):
756 if bbappendlines[i][0] == varname:
757 line = bbappendlines.pop(i)
758 return line
759 return None
760
761 def appendline(varname, op, value):
762 for i in range(0, len(bbappendlines)):
763 item = bbappendlines[i]
764 if item[0] == varname:
765 bbappendlines[i] = (item[0], item[1], item[2] + ' ' + value)
766 break
767 else:
768 bbappendlines.append((varname, op, value))
769
770 destsubdir = rd.getVar('PN')
771 if not update_original_recipe and srcfiles:
772 bbappendlines.append(('FILESEXTRAPATHS:prepend', ':=', '${THISDIR}/${PN}:'))
773
774 appendoverride = ''
775 if machine:
776 bbappendlines.append(('PACKAGE_ARCH', '=', '${MACHINE_ARCH}'))
777 appendoverride = ':%s' % machine
778 copyfiles = {}
779 if srcfiles:
780 instfunclines = []
781 for i, (newfile, param) in enumerate(srcfiles.items()):
782 srcurientry = None
783 if not 'path' in param or not param['path']:
784 if 'newname' in param and param['newname']:
785 srcfile = param['newname']
786 else:
787 srcfile = os.path.basename(newfile)
788 srcurientry = 'file://%s' % srcfile
789 oldentry = None
790 for uri in rd.getVar('SRC_URI').split():
791 if srcurientry in uri:
792 oldentry = uri
793 if params and params[i]:
794 srcurientry = '%s;%s' % (srcurientry, ';'.join('%s=%s' % (k,v) for k,v in params[i].items()))
795 # Double-check it's not there already
796 # FIXME do we care if the entry is added by another bbappend that might go away?
797 if not srcurientry in rd.getVar('SRC_URI').split():
798 if machine:
799 if oldentry:
800 appendline('SRC_URI:remove%s' % appendoverride, '=', ' ' + oldentry)
801 appendline('SRC_URI:append%s' % appendoverride, '=', ' ' + srcurientry)
802 else:
803 if oldentry:
804 if update_original_recipe:
805 removevalues['SRC_URI'] = oldentry
806 else:
807 appendline('SRC_URI:remove', '=', oldentry)
808 appendline('SRC_URI', '+=', srcurientry)
809 param['path'] = srcfile
810 else:
811 srcfile = param['path']
812 copyfiles[newfile] = param
813 if install:
814 institem = install.pop(newfile, None)
815 if institem:
816 (destpath, perms) = institem
817 instdestpath = replace_dir_vars(destpath, rd)
818 instdirline = 'install -d ${D}%s' % os.path.dirname(instdestpath)
819 if not instdirline in instfunclines:
820 instfunclines.append(instdirline)
821 instfunclines.append('install -m %s ${WORKDIR}/%s ${D}%s' % (perms, os.path.basename(srcfile), instdestpath))
822 if instfunclines:
823 bbappendlines.append(('do_install:append%s()' % appendoverride, '', instfunclines))
824
825 if redirect_output:
826 bb.note('Writing append file %s (dry-run)' % appendpath)
827 outfile = os.path.join(redirect_output, os.path.basename(appendpath))
828 # Only take a copy if the file isn't already there (this function may be called
829 # multiple times per operation when we're handling overrides)
830 if os.path.exists(appendpath) and not os.path.exists(outfile):
831 shutil.copy2(appendpath, outfile)
832 elif update_original_recipe:
833 outfile = recipefile
834 else:
835 bb.note('Writing append file %s' % appendpath)
836 outfile = appendpath
837
838 if os.path.exists(outfile):
839 # Work around lack of nonlocal in python 2
840 extvars = {'destsubdir': destsubdir}
841
842 def appendfile_varfunc(varname, origvalue, op, newlines):
843 if varname == 'FILESEXTRAPATHS:prepend':
844 if origvalue.startswith('${THISDIR}/'):
845 popline('FILESEXTRAPATHS:prepend')
846 extvars['destsubdir'] = rd.expand(origvalue.split('${THISDIR}/', 1)[1].rstrip(':'))
847 elif varname == 'PACKAGE_ARCH':
848 if machine:
849 popline('PACKAGE_ARCH')
850 return (machine, None, 4, False)
851 elif varname.startswith('do_install:append'):
852 func = popline(varname)
853 if func:
854 instfunclines = [line.strip() for line in origvalue.strip('\n').splitlines()]
855 for line in func[2]:
856 if not line in instfunclines:
857 instfunclines.append(line)
858 return (instfunclines, None, 4, False)
859 else:
860 splitval = split_var_value(origvalue, assignment=False)
861 changed = False
862 removevar = varname
863 if varname in ['SRC_URI', 'SRC_URI:append%s' % appendoverride]:
864 removevar = 'SRC_URI'
865 line = popline(varname)
866 if line:
867 if line[2] not in splitval:
868 splitval.append(line[2])
869 changed = True
870 else:
871 line = popline(varname)
872 if line:
873 splitval = [line[2]]
874 changed = True
875
876 if removevar in removevalues:
877 remove = removevalues[removevar]
878 if isinstance(remove, str):
879 if remove in splitval:
880 splitval.remove(remove)
881 changed = True
882 else:
883 for removeitem in remove:
884 if removeitem in splitval:
885 splitval.remove(removeitem)
886 changed = True
887
888 if changed:
889 newvalue = splitval
890 if len(newvalue) == 1:
891 # Ensure it's written out as one line
892 if ':append' in varname:
893 newvalue = ' ' + newvalue[0]
894 else:
895 newvalue = newvalue[0]
896 if not newvalue and (op in ['+=', '.='] or ':append' in varname):
897 # There's no point appending nothing
898 newvalue = None
899 if varname.endswith('()'):
900 indent = 4
901 else:
902 indent = -1
903 return (newvalue, None, indent, True)
904 return (origvalue, None, 4, False)
905
906 varnames = [item[0] for item in bbappendlines]
907 if removevalues:
908 varnames.extend(list(removevalues.keys()))
909
910 with open(outfile, 'r') as f:
911 (updated, newlines) = bb.utils.edit_metadata(f, varnames, appendfile_varfunc)
912
913 destsubdir = extvars['destsubdir']
914 else:
915 updated = False
916 newlines = []
917
918 if bbappendlines:
919 for line in bbappendlines:
920 if line[0].endswith('()'):
921 newlines.append('%s {\n %s\n}\n' % (line[0], '\n '.join(line[2])))
922 else:
923 newlines.append('%s %s "%s"\n\n' % line)
924 updated = True
925
926 if updated:
927 with open(outfile, 'w') as f:
928 f.writelines(newlines)
929
930 if copyfiles:
931 if machine:
932 destsubdir = os.path.join(destsubdir, machine)
933 if redirect_output:
934 outdir = redirect_output
935 else:
936 outdir = appenddir
937 for newfile, param in copyfiles.items():
938 srcfile = param['path']
939 patchdir = param.get('patchdir', ".")
940
941 if patchdir != ".":
942 newfile = os.path.join(os.path.split(newfile)[0], patchdir, os.path.split(newfile)[1])
943 filedest = os.path.join(outdir, destsubdir, os.path.basename(srcfile))
944 if os.path.abspath(newfile) != os.path.abspath(filedest):
945 if newfile.startswith(tempfile.gettempdir()):
946 newfiledisp = os.path.basename(newfile)
947 else:
948 newfiledisp = newfile
949 if redirect_output:
950 bb.note('Copying %s to %s (dry-run)' % (newfiledisp, os.path.join(appenddir, destsubdir, os.path.basename(srcfile))))
951 else:
952 bb.note('Copying %s to %s' % (newfiledisp, filedest))
953 bb.utils.mkdirhier(os.path.dirname(filedest))
954 shutil.copyfile(newfile, filedest)
955
956 return (appendpath, os.path.join(appenddir, destsubdir))
957
958
959def find_layerdir(fn):
960 """ Figure out the path to the base of the layer containing a file (e.g. a recipe)"""
961 pth = os.path.abspath(fn)
962 layerdir = ''
963 while pth:
964 if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')):
965 layerdir = pth
966 break
967 pth = os.path.dirname(pth)
968 if pth == '/':
969 return None
970 return layerdir
971
972
973def replace_dir_vars(path, d):
974 """Replace common directory paths with appropriate variable references (e.g. /etc becomes ${sysconfdir})"""
975 dirvars = {}
976 # Sort by length so we get the variables we're interested in first
977 for var in sorted(list(d.keys()), key=len):
978 if var.endswith('dir') and var.lower() == var:
979 value = d.getVar(var)
980 if value.startswith('/') and not '\n' in value and value not in dirvars:
981 dirvars[value] = var
982 for dirpath in sorted(list(dirvars.keys()), reverse=True):
983 path = path.replace(dirpath, '${%s}' % dirvars[dirpath])
984 return path
985
986def get_recipe_pv_with_pfx_sfx(pv, uri_type):
987 """
988 Get PV separating prefix and suffix components.
989
990 Returns tuple with pv, prefix and suffix.
991 """
992 pfx = ''
993 sfx = ''
994
995 if uri_type == 'git':
996 git_regex = re.compile(r"(?P<pfx>v?)(?P<ver>.*?)(?P<sfx>\+[^\+]*(git)?r?(AUTOINC\+)?)(?P<rev>.*)")
997 m = git_regex.match(pv)
998
999 if m:
1000 pv = m.group('ver')
1001 pfx = m.group('pfx')
1002 sfx = m.group('sfx')
1003 else:
1004 regex = re.compile(r"(?P<pfx>(v|r)?)(?P<ver>.*)")
1005 m = regex.match(pv)
1006 if m:
1007 pv = m.group('ver')
1008 pfx = m.group('pfx')
1009
1010 return (pv, pfx, sfx)
1011
1012def get_recipe_upstream_version(rd):
1013 """
1014 Get upstream version of recipe using bb.fetch2 methods with support for
1015 http, https, ftp and git.
1016
1017 bb.fetch2 exceptions can be raised,
1018 FetchError when don't have network access or upstream site don't response.
1019 NoMethodError when uri latest_versionstring method isn't implemented.
1020
1021 Returns a dictonary with version, repository revision, current_version, type and datetime.
1022 Type can be A for Automatic, M for Manual and U for Unknown.
1023 """
1024 from bb.fetch2 import decodeurl
1025 from datetime import datetime
1026
1027 ru = {}
1028 ru['current_version'] = rd.getVar('PV')
1029 ru['version'] = ''
1030 ru['type'] = 'U'
1031 ru['datetime'] = ''
1032 ru['revision'] = ''
1033
1034 # XXX: If don't have SRC_URI means that don't have upstream sources so
1035 # returns the current recipe version, so that upstream version check
1036 # declares a match.
1037 src_uris = rd.getVar('SRC_URI')
1038 if not src_uris:
1039 ru['version'] = ru['current_version']
1040 ru['type'] = 'M'
1041 ru['datetime'] = datetime.now()
1042 return ru
1043
1044 # XXX: we suppose that the first entry points to the upstream sources
1045 src_uri = src_uris.split()[0]
1046 uri_type, _, _, _, _, _ = decodeurl(src_uri)
1047
1048 (pv, pfx, sfx) = get_recipe_pv_with_pfx_sfx(rd.getVar('PV'), uri_type)
1049 ru['current_version'] = pv
1050
1051 manual_upstream_version = rd.getVar("RECIPE_UPSTREAM_VERSION")
1052 if manual_upstream_version:
1053 # manual tracking of upstream version.
1054 ru['version'] = manual_upstream_version
1055 ru['type'] = 'M'
1056
1057 manual_upstream_date = rd.getVar("CHECK_DATE")
1058 if manual_upstream_date:
1059 date = datetime.strptime(manual_upstream_date, "%b %d, %Y")
1060 else:
1061 date = datetime.now()
1062 ru['datetime'] = date
1063
1064 elif uri_type == "file":
1065 # files are always up-to-date
1066 ru['version'] = pv
1067 ru['type'] = 'A'
1068 ru['datetime'] = datetime.now()
1069 else:
1070 ud = bb.fetch2.FetchData(src_uri, rd)
1071 if rd.getVar("UPSTREAM_CHECK_COMMITS") == "1":
1072 bb.fetch2.get_srcrev(rd)
1073 revision = ud.method.latest_revision(ud, rd, 'default')
1074 upversion = pv
1075 if revision != rd.getVar("SRCREV"):
1076 upversion = upversion + "-new-commits-available"
1077 else:
1078 pupver = ud.method.latest_versionstring(ud, rd)
1079 (upversion, revision) = pupver
1080
1081 if upversion:
1082 ru['version'] = upversion
1083 ru['type'] = 'A'
1084
1085 if revision:
1086 ru['revision'] = revision
1087
1088 ru['datetime'] = datetime.now()
1089
1090 return ru
1091
1092def _get_recipe_upgrade_status(data):
1093 uv = get_recipe_upstream_version(data)
1094
1095 pn = data.getVar('PN')
1096 cur_ver = uv['current_version']
1097
1098 upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN')
1099 if not uv['version']:
1100 status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
1101 else:
1102 cmp = vercmp_string(uv['current_version'], uv['version'])
1103 if cmp == -1:
1104 status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN"
1105 elif cmp == 0:
1106 status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN"
1107 else:
1108 status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
1109
1110 next_ver = uv['version'] if uv['version'] else "N/A"
1111 revision = uv['revision'] if uv['revision'] else "N/A"
1112 maintainer = data.getVar('RECIPE_MAINTAINER')
1113 no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON')
1114
1115 return (pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason)
1116
1117def get_recipe_upgrade_status(recipes=None):
1118 pkgs_list = []
1119 data_copy_list = []
1120 copy_vars = ('SRC_URI',
1121 'PV',
1122 'DL_DIR',
1123 'PN',
1124 'CACHE',
1125 'PERSISTENT_DIR',
1126 'BB_URI_HEADREVS',
1127 'UPSTREAM_CHECK_COMMITS',
1128 'UPSTREAM_CHECK_GITTAGREGEX',
1129 'UPSTREAM_CHECK_REGEX',
1130 'UPSTREAM_CHECK_URI',
1131 'UPSTREAM_VERSION_UNKNOWN',
1132 'RECIPE_MAINTAINER',
1133 'RECIPE_NO_UPDATE_REASON',
1134 'RECIPE_UPSTREAM_VERSION',
1135 'RECIPE_UPSTREAM_DATE',
1136 'CHECK_DATE',
1137 'FETCHCMD_bzr',
1138 'FETCHCMD_ccrc',
1139 'FETCHCMD_cvs',
1140 'FETCHCMD_git',
1141 'FETCHCMD_hg',
1142 'FETCHCMD_npm',
1143 'FETCHCMD_osc',
1144 'FETCHCMD_p4',
1145 'FETCHCMD_repo',
1146 'FETCHCMD_s3',
1147 'FETCHCMD_svn',
1148 'FETCHCMD_wget',
1149 )
1150
1151 with bb.tinfoil.Tinfoil() as tinfoil:
1152 tinfoil.prepare(config_only=False)
1153
1154 if not recipes:
1155 recipes = tinfoil.all_recipe_files(variants=False)
1156
1157 for fn in recipes:
1158 try:
1159 if fn.startswith("/"):
1160 data = tinfoil.parse_recipe_file(fn)
1161 else:
1162 data = tinfoil.parse_recipe(fn)
1163 except bb.providers.NoProvider:
1164 bb.note(" No provider for %s" % fn)
1165 continue
1166
1167 unreliable = data.getVar('UPSTREAM_CHECK_UNRELIABLE')
1168 if unreliable == "1":
1169 bb.note(" Skip package %s as upstream check unreliable" % pn)
1170 continue
1171
1172 data_copy = bb.data.init()
1173 for var in copy_vars:
1174 data_copy.setVar(var, data.getVar(var))
1175 for k in data:
1176 if k.startswith('SRCREV'):
1177 data_copy.setVar(k, data.getVar(k))
1178
1179 data_copy_list.append(data_copy)
1180
1181 from concurrent.futures import ProcessPoolExecutor
1182 with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor:
1183 pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list)
1184
1185 return pkgs_list
diff --git a/meta-xilinx-core/lib/oe/reproducible.py b/meta-xilinx-core/lib/oe/reproducible.py
new file mode 100644
index 00000000..448befce
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/reproducible.py
@@ -0,0 +1,197 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6import os
7import subprocess
8import bb
9
10# For reproducible builds, this code sets the default SOURCE_DATE_EPOCH in each
11# component's build environment. The format is number of seconds since the
12# system epoch.
13#
14# Upstream components (generally) respect this environment variable,
15# using it in place of the "current" date and time.
16# See https://reproducible-builds.org/specs/source-date-epoch/
17#
18# The default value of SOURCE_DATE_EPOCH comes from the function
19# get_source_date_epoch_value which reads from the SDE_FILE, or if the file
20# is not available will use the fallback of SOURCE_DATE_EPOCH_FALLBACK.
21#
22# The SDE_FILE is normally constructed from the function
23# create_source_date_epoch_stamp which is typically added as a postfuncs to
24# the do_unpack task. If a recipe does NOT have do_unpack, it should be added
25# to a task that runs after the source is available and before the
26# do_deploy_source_date_epoch task is executed.
27#
28# If a recipe wishes to override the default behavior it should set it's own
29# SOURCE_DATE_EPOCH or override the do_deploy_source_date_epoch_stamp task
30# with recipe-specific functionality to write the appropriate
31# SOURCE_DATE_EPOCH into the SDE_FILE.
32#
33# SOURCE_DATE_EPOCH is intended to be a reproducible value. This value should
34# be reproducible for anyone who builds the same revision from the same
35# sources.
36#
37# There are 4 ways the create_source_date_epoch_stamp function determines what
38# becomes SOURCE_DATE_EPOCH:
39#
40# 1. Use the value from __source_date_epoch.txt file if this file exists.
41# This file was most likely created in the previous build by one of the
42# following methods 2,3,4.
43# Alternatively, it can be provided by a recipe via SRC_URI.
44#
45# If the file does not exist:
46#
47# 2. If there is a git checkout, use the last git commit timestamp.
48# Git does not preserve file timestamps on checkout.
49#
50# 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ...
51# This works for well-kept repositories distributed via tarball.
52#
53# 4. Use the modification time of the youngest file in the source tree, if
54# there is one.
55# This will be the newest file from the distribution tarball, if any.
56#
57# 5. Fall back to a fixed timestamp (SOURCE_DATE_EPOCH_FALLBACK).
58#
59# Once the value is determined, it is stored in the recipe's SDE_FILE.
60
61def get_source_date_epoch_from_known_files(d, sourcedir):
62 source_date_epoch = None
63 newest_file = None
64 known_files = set(["NEWS", "ChangeLog", "Changelog", "CHANGES"])
65 for file in known_files:
66 filepath = os.path.join(sourcedir, file)
67 if os.path.isfile(filepath):
68 mtime = int(os.lstat(filepath).st_mtime)
69 # There may be more than one "known_file" present, if so, use the youngest one
70 if not source_date_epoch or mtime > source_date_epoch:
71 source_date_epoch = mtime
72 newest_file = filepath
73 if newest_file:
74 bb.debug(1, "SOURCE_DATE_EPOCH taken from: %s" % newest_file)
75 return source_date_epoch
76
77def find_git_folder(d, sourcedir):
78 # First guess: WORKDIR/git
79 # This is the default git fetcher unpack path
80 workdir = d.getVar('WORKDIR')
81 gitpath = os.path.join(workdir, "git/.git")
82 if os.path.isdir(gitpath):
83 return gitpath
84
85 # Second guess: ${S}
86 gitpath = os.path.join(sourcedir, ".git")
87 if os.path.isdir(gitpath):
88 return gitpath
89
90 # Perhaps there was a subpath or destsuffix specified.
91 # Go looking in the WORKDIR
92 exclude = set(["build", "image", "license-destdir", "patches", "pseudo",
93 "recipe-sysroot", "recipe-sysroot-native", "sysroot-destdir", "temp"])
94 for root, dirs, files in os.walk(workdir, topdown=True):
95 dirs[:] = [d for d in dirs if d not in exclude]
96 if '.git' in dirs:
97 return os.path.join(root, ".git")
98
99 bb.warn("Failed to find a git repository in WORKDIR: %s" % workdir)
100 return None
101
102def get_source_date_epoch_from_git(d, sourcedir):
103 if not "git://" in d.getVar('SRC_URI') and not "gitsm://" in d.getVar('SRC_URI'):
104 return None
105
106 gitpath = find_git_folder(d, sourcedir)
107 if not gitpath:
108 return None
109
110 # Check that the repository has a valid HEAD; it may not if subdir is used
111 # in SRC_URI
112 p = subprocess.run(['git', '--git-dir', gitpath, 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
113 if p.returncode != 0:
114 bb.debug(1, "%s does not have a valid HEAD: %s" % (gitpath, p.stdout.decode('utf-8')))
115 return None
116
117 bb.debug(1, "git repository: %s" % gitpath)
118 p = subprocess.run(['git', '-c', 'log.showSignature=false', '--git-dir', gitpath, 'log', '-1', '--pretty=%ct'],
119 check=True, stdout=subprocess.PIPE)
120 return int(p.stdout.decode('utf-8'))
121
122def get_source_date_epoch_from_youngest_file(d, sourcedir):
123 if sourcedir == d.getVar('WORKDIR'):
124 # These sources are almost certainly not from a tarball
125 return None
126
127 # Do it the hard way: check all files and find the youngest one...
128 source_date_epoch = None
129 newest_file = None
130 for root, dirs, files in os.walk(sourcedir, topdown=True):
131 files = [f for f in files if not f[0] == '.']
132
133 for fname in files:
134 if fname == "singletask.lock":
135 # Ignore externalsrc/devtool lockfile [YOCTO #14921]
136 continue
137 filename = os.path.join(root, fname)
138 try:
139 mtime = int(os.lstat(filename).st_mtime)
140 except ValueError:
141 mtime = 0
142 if not source_date_epoch or mtime > source_date_epoch:
143 source_date_epoch = mtime
144 newest_file = filename
145
146 if newest_file:
147 bb.debug(1, "Newest file found: %s" % newest_file)
148 return source_date_epoch
149
150def fixed_source_date_epoch(d):
151 bb.debug(1, "No tarball or git repo found to determine SOURCE_DATE_EPOCH")
152 source_date_epoch = d.getVar('SOURCE_DATE_EPOCH_FALLBACK')
153 if source_date_epoch:
154 bb.debug(1, "Using SOURCE_DATE_EPOCH_FALLBACK")
155 return int(source_date_epoch)
156 return 0
157
158def get_source_date_epoch(d, sourcedir):
159 return (
160 get_source_date_epoch_from_git(d, sourcedir) or
161 get_source_date_epoch_from_youngest_file(d, sourcedir) or
162 fixed_source_date_epoch(d) # Last resort
163 )
164
165def epochfile_read(epochfile, d):
166 cached, efile = d.getVar('__CACHED_SOURCE_DATE_EPOCH') or (None, None)
167 if cached and efile == epochfile:
168 return cached
169
170 if cached and epochfile != efile:
171 bb.debug(1, "Epoch file changed from %s to %s" % (efile, epochfile))
172
173 source_date_epoch = int(d.getVar('SOURCE_DATE_EPOCH_FALLBACK'))
174 try:
175 with open(epochfile, 'r') as f:
176 s = f.read()
177 try:
178 source_date_epoch = int(s)
179 except ValueError:
180 bb.warn("SOURCE_DATE_EPOCH value '%s' is invalid. Reverting to SOURCE_DATE_EPOCH_FALLBACK" % s)
181 source_date_epoch = int(d.getVar('SOURCE_DATE_EPOCH_FALLBACK'))
182 bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch)
183 except FileNotFoundError:
184 bb.debug(1, "Cannot find %s. SOURCE_DATE_EPOCH will default to %d" % (epochfile, source_date_epoch))
185
186 d.setVar('__CACHED_SOURCE_DATE_EPOCH', (str(source_date_epoch), epochfile))
187 return str(source_date_epoch)
188
189def epochfile_write(source_date_epoch, epochfile, d):
190
191 bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch)
192 bb.utils.mkdirhier(os.path.dirname(epochfile))
193
194 tmp_file = "%s.new" % epochfile
195 with open(tmp_file, 'w') as f:
196 f.write(str(source_date_epoch))
197 os.rename(tmp_file, epochfile)
diff --git a/meta-xilinx-core/lib/oe/rootfs.py b/meta-xilinx-core/lib/oe/rootfs.py
new file mode 100644
index 00000000..5abce4ad
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/rootfs.py
@@ -0,0 +1,438 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6from abc import ABCMeta, abstractmethod
7from oe.utils import execute_pre_post_process
8from oe.package_manager import *
9from oe.manifest import *
10import oe.path
11import shutil
12import os
13import subprocess
14import re
15
16class Rootfs(object, metaclass=ABCMeta):
17 """
18 This is an abstract class. Do not instantiate this directly.
19 """
20
21 def __init__(self, d, progress_reporter=None, logcatcher=None):
22 self.d = d
23 self.pm = None
24 self.image_rootfs = self.d.getVar('IMAGE_ROOTFS')
25 self.deploydir = self.d.getVar('IMGDEPLOYDIR')
26 self.progress_reporter = progress_reporter
27 self.logcatcher = logcatcher
28
29 self.install_order = Manifest.INSTALL_ORDER
30
31 @abstractmethod
32 def _create(self):
33 pass
34
35 @abstractmethod
36 def _get_delayed_postinsts(self):
37 pass
38
39 @abstractmethod
40 def _save_postinsts(self):
41 pass
42
43 @abstractmethod
44 def _log_check(self):
45 pass
46
47 def _log_check_common(self, type, match):
48 # Ignore any lines containing log_check to avoid recursion, and ignore
49 # lines beginning with a + since sh -x may emit code which isn't
50 # actually executed, but may contain error messages
51 excludes = [ 'log_check', r'^\+' ]
52 if hasattr(self, 'log_check_expected_regexes'):
53 excludes.extend(self.log_check_expected_regexes)
54 # Insert custom log_check excludes
55 excludes += [x for x in (self.d.getVar("IMAGE_LOG_CHECK_EXCLUDES") or "").split(" ") if x]
56 excludes = [re.compile(x) for x in excludes]
57 r = re.compile(match)
58 log_path = self.d.expand("${T}/log.do_rootfs")
59 messages = []
60 with open(log_path, 'r') as log:
61 for line in log:
62 if self.logcatcher and self.logcatcher.contains(line.rstrip()):
63 continue
64 for ee in excludes:
65 m = ee.search(line)
66 if m:
67 break
68 if m:
69 continue
70
71 m = r.search(line)
72 if m:
73 messages.append('[log_check] %s' % line)
74 if messages:
75 if len(messages) == 1:
76 msg = '1 %s message' % type
77 else:
78 msg = '%d %s messages' % (len(messages), type)
79 msg = '[log_check] %s: found %s in the logfile:\n%s' % \
80 (self.d.getVar('PN'), msg, ''.join(messages))
81 if type == 'error':
82 bb.fatal(msg)
83 else:
84 bb.warn(msg)
85
86 def _log_check_warn(self):
87 self._log_check_common('warning', '^(warn|Warn|WARNING:)')
88
89 def _log_check_error(self):
90 self._log_check_common('error', self.log_check_regex)
91
92 def _insert_feed_uris(self):
93 if bb.utils.contains("IMAGE_FEATURES", "package-management",
94 True, False, self.d):
95 self.pm.insert_feeds_uris(self.d.getVar('PACKAGE_FEED_URIS') or "",
96 self.d.getVar('PACKAGE_FEED_BASE_PATHS') or "",
97 self.d.getVar('PACKAGE_FEED_ARCHS'))
98
99
100 """
101 The _cleanup() method should be used to clean-up stuff that we don't really
102 want to end up on target. For example, in the case of RPM, the DB locks.
103 The method is called, once, at the end of create() method.
104 """
105 @abstractmethod
106 def _cleanup(self):
107 pass
108
109 def _setup_dbg_rootfs(self, package_paths):
110 gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0'
111 if gen_debugfs != '1':
112 return
113
114 bb.note(" Renaming the original rootfs...")
115 try:
116 shutil.rmtree(self.image_rootfs + '-orig')
117 except:
118 pass
119 bb.utils.rename(self.image_rootfs, self.image_rootfs + '-orig')
120
121 bb.note(" Creating debug rootfs...")
122 bb.utils.mkdirhier(self.image_rootfs)
123
124 bb.note(" Copying back package database...")
125 for path in package_paths:
126 bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(path))
127 if os.path.isdir(self.image_rootfs + '-orig' + path):
128 shutil.copytree(self.image_rootfs + '-orig' + path, self.image_rootfs + path, symlinks=True)
129 elif os.path.isfile(self.image_rootfs + '-orig' + path):
130 shutil.copyfile(self.image_rootfs + '-orig' + path, self.image_rootfs + path)
131
132 # Copy files located in /usr/lib/debug or /usr/src/debug
133 for dir in ["/usr/lib/debug", "/usr/src/debug"]:
134 src = self.image_rootfs + '-orig' + dir
135 if os.path.exists(src):
136 dst = self.image_rootfs + dir
137 bb.utils.mkdirhier(os.path.dirname(dst))
138 shutil.copytree(src, dst)
139
140 # Copy files with suffix '.debug' or located in '.debug' dir.
141 for root, dirs, files in os.walk(self.image_rootfs + '-orig'):
142 relative_dir = root[len(self.image_rootfs + '-orig'):]
143 for f in files:
144 if f.endswith('.debug') or '/.debug' in relative_dir:
145 bb.utils.mkdirhier(self.image_rootfs + relative_dir)
146 shutil.copy(os.path.join(root, f),
147 self.image_rootfs + relative_dir)
148
149 bb.note(" Install complementary '*-dbg' packages...")
150 self.pm.install_complementary('*-dbg')
151
152 if self.d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg':
153 bb.note(" Install complementary '*-src' packages...")
154 self.pm.install_complementary('*-src')
155
156 """
157 Install additional debug packages. Possibility to install additional packages,
158 which are not automatically installed as complementary package of
159 standard one, e.g. debug package of static libraries.
160 """
161 extra_debug_pkgs = self.d.getVar('IMAGE_INSTALL_DEBUGFS')
162 if extra_debug_pkgs:
163 bb.note(" Install extra debug packages...")
164 self.pm.install(extra_debug_pkgs.split(), True)
165
166 bb.note(" Removing package database...")
167 for path in package_paths:
168 if os.path.isdir(self.image_rootfs + path):
169 shutil.rmtree(self.image_rootfs + path)
170 elif os.path.isfile(self.image_rootfs + path):
171 os.remove(self.image_rootfs + path)
172
173 bb.note(" Rename debug rootfs...")
174 try:
175 shutil.rmtree(self.image_rootfs + '-dbg')
176 except:
177 pass
178 bb.utils.rename(self.image_rootfs, self.image_rootfs + '-dbg')
179
180 bb.note(" Restoring original rootfs...")
181 bb.utils.rename(self.image_rootfs + '-orig', self.image_rootfs)
182
183 def _exec_shell_cmd(self, cmd):
184 try:
185 subprocess.check_output(cmd, stderr=subprocess.STDOUT)
186 except subprocess.CalledProcessError as e:
187 return("Command '%s' returned %d:\n%s" % (e.cmd, e.returncode, e.output))
188
189 return None
190
191 def create(self):
192 bb.note("###### Generate rootfs #######")
193 pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND")
194 post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND")
195 rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND')
196
197 def make_last(command, commands):
198 commands = commands.split()
199 if command in commands:
200 commands.remove(command)
201 commands.append(command)
202 return "".join(commands)
203
204 # We want this to run as late as possible, in particular after
205 # systemd_sysusers_create and set_user_group. Using :append is not enough
206 make_last("tidy_shadowutils_files", post_process_cmds)
207 make_last("rootfs_reproducible", post_process_cmds)
208
209 execute_pre_post_process(self.d, pre_process_cmds)
210
211 if self.progress_reporter:
212 self.progress_reporter.next_stage()
213
214 # call the package manager dependent create method
215 self._create()
216
217 sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir')
218 bb.utils.mkdirhier(sysconfdir)
219 with open(sysconfdir + "/version", "w+") as ver:
220 ver.write(self.d.getVar('BUILDNAME') + "\n")
221
222 execute_pre_post_process(self.d, rootfs_post_install_cmds)
223
224 self.pm.run_intercepts()
225
226 execute_pre_post_process(self.d, post_process_cmds)
227
228 if self.progress_reporter:
229 self.progress_reporter.next_stage()
230
231 if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs",
232 True, False, self.d) and \
233 not bb.utils.contains("IMAGE_FEATURES",
234 "read-only-rootfs-delayed-postinsts",
235 True, False, self.d):
236 delayed_postinsts = self._get_delayed_postinsts()
237 if delayed_postinsts is not None:
238 bb.fatal("The following packages could not be configured "
239 "offline and rootfs is read-only: %s" %
240 delayed_postinsts)
241
242 if self.d.getVar('USE_DEVFS') != "1":
243 self._create_devfs()
244
245 self._uninstall_unneeded()
246
247 if self.progress_reporter:
248 self.progress_reporter.next_stage()
249
250 self._insert_feed_uris()
251
252 self._run_ldconfig()
253
254 if self.d.getVar('USE_DEPMOD') != "0":
255 self._generate_kernel_module_deps()
256
257 self._cleanup()
258 self._log_check()
259
260 if self.progress_reporter:
261 self.progress_reporter.next_stage()
262
263
264 def _uninstall_unneeded(self):
265 # Remove the run-postinsts package if no delayed postinsts are found
266 delayed_postinsts = self._get_delayed_postinsts()
267 if delayed_postinsts is None:
268 if os.path.exists(self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/init.d/run-postinsts")) or os.path.exists(self.d.expand("${IMAGE_ROOTFS}${systemd_system_unitdir}/run-postinsts.service")):
269 self.pm.remove(["run-postinsts"])
270
271 image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs",
272 True, False, self.d) and \
273 not bb.utils.contains("IMAGE_FEATURES",
274 "read-only-rootfs-delayed-postinsts",
275 True, False, self.d)
276
277 image_rorfs_force = self.d.getVar('FORCE_RO_REMOVE')
278
279 if image_rorfs or image_rorfs_force == "1":
280 # Remove components that we don't need if it's a read-only rootfs
281 unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED").split()
282 pkgs_installed = image_list_installed_packages(self.d)
283 # Make sure update-alternatives is removed last. This is
284 # because its database has to available while uninstalling
285 # other packages, allowing alternative symlinks of packages
286 # to be uninstalled or to be managed correctly otherwise.
287 provider = self.d.getVar("VIRTUAL-RUNTIME_update-alternatives")
288 pkgs_to_remove = sorted([pkg for pkg in pkgs_installed if pkg in unneeded_pkgs], key=lambda x: x == provider)
289
290 # update-alternatives provider is removed in its own remove()
291 # call because all package managers do not guarantee the packages
292 # are removed in the order they given in the list (which is
293 # passed to the command line). The sorting done earlier is
294 # utilized to implement the 2-stage removal.
295 if len(pkgs_to_remove) > 1:
296 self.pm.remove(pkgs_to_remove[:-1], False)
297 if len(pkgs_to_remove) > 0:
298 self.pm.remove([pkgs_to_remove[-1]], False)
299
300 if delayed_postinsts:
301 self._save_postinsts()
302 if image_rorfs:
303 bb.warn("There are post install scripts "
304 "in a read-only rootfs")
305
306 post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND")
307 execute_pre_post_process(self.d, post_uninstall_cmds)
308
309 runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management",
310 True, False, self.d)
311 if not runtime_pkgmanage:
312 # Remove the package manager data files
313 self.pm.remove_packaging_data()
314
315 def _run_ldconfig(self):
316 if self.d.getVar('LDCONFIGDEPEND'):
317 bb.note("Executing: ldconfig -r " + self.image_rootfs + " -c new -v -X")
318 self._exec_shell_cmd(['ldconfig', '-r', self.image_rootfs, '-c',
319 'new', '-v', '-X'])
320
321 image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs",
322 True, False, self.d)
323 ldconfig_in_features = bb.utils.contains("DISTRO_FEATURES", "ldconfig",
324 True, False, self.d)
325 if image_rorfs or not ldconfig_in_features:
326 ldconfig_cache_dir = os.path.join(self.image_rootfs, "var/cache/ldconfig")
327 if os.path.exists(ldconfig_cache_dir):
328 bb.note("Removing ldconfig auxiliary cache...")
329 shutil.rmtree(ldconfig_cache_dir)
330
331 def _check_for_kernel_modules(self, modules_dir):
332 for root, dirs, files in os.walk(modules_dir, topdown=True):
333 for name in files:
334 found_ko = name.endswith((".ko", ".ko.gz", ".ko.xz", ".ko.zst"))
335 if found_ko:
336 return found_ko
337 return False
338
339 def _generate_kernel_module_deps(self):
340 modules_dir = os.path.join(self.image_rootfs, 'lib', 'modules')
341 # if we don't have any modules don't bother to do the depmod
342 if not self._check_for_kernel_modules(modules_dir):
343 bb.note("No Kernel Modules found, not running depmod")
344 return
345
346 pkgdatadir = self.d.getVar('PKGDATA_DIR')
347
348 # PKGDATA_DIR can include multiple kernels so we run depmod for each
349 # one of them.
350 for direntry in os.listdir(pkgdatadir):
351 match = re.match('(.*)-depmod', direntry)
352 if not match:
353 continue
354 kernel_package_name = match.group(1)
355
356 kernel_abi_ver_file = oe.path.join(pkgdatadir, direntry, kernel_package_name + '-abiversion')
357 if not os.path.exists(kernel_abi_ver_file):
358 bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file)
359
360 with open(kernel_abi_ver_file) as f:
361 kernel_ver = f.read().strip(' \n')
362
363 versioned_modules_dir = os.path.join(self.image_rootfs, modules_dir, kernel_ver)
364
365 bb.utils.mkdirhier(versioned_modules_dir)
366
367 bb.note("Running depmodwrapper for %s ..." % versioned_modules_dir)
368 if self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, kernel_ver, kernel_package_name]):
369 bb.fatal("Kernel modules dependency generation failed")
370
371 """
372 Create devfs:
373 * IMAGE_DEVICE_TABLE is the old name to an absolute path to a device table file
374 * IMAGE_DEVICE_TABLES is a new name for a file, or list of files, seached
375 for in the BBPATH
376 If neither are specified then the default name of files/device_table-minimal.txt
377 is searched for in the BBPATH (same as the old version.)
378 """
379 def _create_devfs(self):
380 devtable_list = []
381 devtable = self.d.getVar('IMAGE_DEVICE_TABLE')
382 if devtable is not None:
383 devtable_list.append(devtable)
384 else:
385 devtables = self.d.getVar('IMAGE_DEVICE_TABLES')
386 if devtables is None:
387 devtables = 'files/device_table-minimal.txt'
388 for devtable in devtables.split():
389 devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH'), devtable))
390
391 for devtable in devtable_list:
392 self._exec_shell_cmd(["makedevs", "-r",
393 self.image_rootfs, "-D", devtable])
394
395
396def get_class_for_type(imgtype):
397 import importlib
398 mod = importlib.import_module('oe.package_manager.' + imgtype + '.rootfs')
399 return mod.PkgRootfs
400
401def variable_depends(d, manifest_dir=None):
402 img_type = d.getVar('IMAGE_PKGTYPE')
403 cls = get_class_for_type(img_type)
404 return cls._depends_list()
405
406def create_rootfs(d, manifest_dir=None, progress_reporter=None, logcatcher=None):
407 env_bkp = os.environ.copy()
408
409 img_type = d.getVar('IMAGE_PKGTYPE')
410
411 cls = get_class_for_type(img_type)
412 cls(d, manifest_dir, progress_reporter, logcatcher).create()
413 os.environ.clear()
414 os.environ.update(env_bkp)
415
416
417def image_list_installed_packages(d, rootfs_dir=None):
418 # Theres no rootfs for baremetal images
419 if bb.data.inherits_class('baremetal-image', d):
420 return ""
421
422 if not rootfs_dir:
423 rootfs_dir = d.getVar('IMAGE_ROOTFS')
424
425 img_type = d.getVar('IMAGE_PKGTYPE')
426
427 import importlib
428 cls = importlib.import_module('oe.package_manager.' + img_type)
429 return cls.PMPkgsList(d, rootfs_dir).list_pkgs()
430
431if __name__ == "__main__":
432 """
433 We should be able to run this as a standalone script, from outside bitbake
434 environment.
435 """
436 """
437 TBD
438 """
diff --git a/meta-xilinx-core/lib/oe/rust.py b/meta-xilinx-core/lib/oe/rust.py
new file mode 100644
index 00000000..185553ee
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/rust.py
@@ -0,0 +1,13 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: MIT
5#
6
7# Handle mismatches between `uname -m`-style output and Rust's arch names
8def arch_to_rust_arch(arch):
9 if arch == "ppc64le":
10 return "powerpc64le"
11 if arch in ('riscv32', 'riscv64'):
12 return arch + 'gc'
13 return arch
diff --git a/meta-xilinx-core/lib/oe/sbom.py b/meta-xilinx-core/lib/oe/sbom.py
new file mode 100644
index 00000000..fd4b6895
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/sbom.py
@@ -0,0 +1,120 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import collections
8
9DepRecipe = collections.namedtuple("DepRecipe", ("doc", "doc_sha1", "recipe"))
10DepSource = collections.namedtuple("DepSource", ("doc", "doc_sha1", "recipe", "file"))
11
12
13def get_recipe_spdxid(d):
14 return "SPDXRef-%s-%s" % ("Recipe", d.getVar("PN"))
15
16
17def get_download_spdxid(d, idx):
18 return "SPDXRef-Download-%s-%d" % (d.getVar("PN"), idx)
19
20
21def get_package_spdxid(pkg):
22 return "SPDXRef-Package-%s" % pkg
23
24
25def get_source_file_spdxid(d, idx):
26 return "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), idx)
27
28
29def get_packaged_file_spdxid(pkg, idx):
30 return "SPDXRef-PackagedFile-%s-%d" % (pkg, idx)
31
32
33def get_image_spdxid(img):
34 return "SPDXRef-Image-%s" % img
35
36
37def get_sdk_spdxid(sdk):
38 return "SPDXRef-SDK-%s" % sdk
39
40
41def _doc_path_by_namespace(spdx_deploy, arch, doc_namespace):
42 return spdx_deploy / "by-namespace" / arch / doc_namespace.replace("/", "_")
43
44
45def doc_find_by_namespace(spdx_deploy, search_arches, doc_namespace):
46 for pkgarch in search_arches:
47 p = _doc_path_by_namespace(spdx_deploy, pkgarch, doc_namespace)
48 if os.path.exists(p):
49 return p
50 return None
51
52
53def _doc_path_by_hashfn(spdx_deploy, arch, doc_name, hashfn):
54 return (
55 spdx_deploy / "by-hash" / arch / hashfn.split()[1] / (doc_name + ".spdx.json")
56 )
57
58
59def doc_find_by_hashfn(spdx_deploy, search_arches, doc_name, hashfn):
60 for pkgarch in search_arches:
61 p = _doc_path_by_hashfn(spdx_deploy, pkgarch, doc_name, hashfn)
62 if os.path.exists(p):
63 return p
64 return None
65
66
67def doc_path(spdx_deploy, doc_name, arch, subdir):
68 return spdx_deploy / arch / subdir / (doc_name + ".spdx.json")
69
70
71def write_doc(d, spdx_doc, arch, subdir, spdx_deploy=None, indent=None):
72 from pathlib import Path
73
74 if spdx_deploy is None:
75 spdx_deploy = Path(d.getVar("SPDXDEPLOY"))
76
77 dest = doc_path(spdx_deploy, spdx_doc.name, arch, subdir)
78 dest.parent.mkdir(exist_ok=True, parents=True)
79 with dest.open("wb") as f:
80 doc_sha1 = spdx_doc.to_json(f, sort_keys=True, indent=indent)
81
82 l = _doc_path_by_namespace(spdx_deploy, arch, spdx_doc.documentNamespace)
83 l.parent.mkdir(exist_ok=True, parents=True)
84 l.symlink_to(os.path.relpath(dest, l.parent))
85
86 l = _doc_path_by_hashfn(
87 spdx_deploy, arch, spdx_doc.name, d.getVar("BB_HASHFILENAME")
88 )
89 l.parent.mkdir(exist_ok=True, parents=True)
90 l.symlink_to(os.path.relpath(dest, l.parent))
91
92 return doc_sha1
93
94
95def read_doc(fn):
96 import hashlib
97 import oe.spdx
98 import io
99 import contextlib
100
101 @contextlib.contextmanager
102 def get_file():
103 if isinstance(fn, io.IOBase):
104 yield fn
105 else:
106 with fn.open("rb") as f:
107 yield f
108
109 with get_file() as f:
110 sha1 = hashlib.sha1()
111 while True:
112 chunk = f.read(4096)
113 if not chunk:
114 break
115 sha1.update(chunk)
116
117 f.seek(0)
118 doc = oe.spdx.SPDXDocument.from_json(f)
119
120 return (doc, sha1.hexdigest())
diff --git a/meta-xilinx-core/lib/oe/sdk.py b/meta-xilinx-core/lib/oe/sdk.py
new file mode 100644
index 00000000..3dc36722
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/sdk.py
@@ -0,0 +1,160 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7from abc import ABCMeta, abstractmethod
8from oe.utils import execute_pre_post_process
9from oe.manifest import *
10from oe.package_manager import *
11import os
12import traceback
13
14class Sdk(object, metaclass=ABCMeta):
15 def __init__(self, d, manifest_dir):
16 self.d = d
17 self.sdk_output = self.d.getVar('SDK_OUTPUT')
18 self.sdk_native_path = self.d.getVar('SDKPATHNATIVE').strip('/')
19 self.target_path = self.d.getVar('SDKTARGETSYSROOT').strip('/')
20 self.sysconfdir = self.d.getVar('sysconfdir').strip('/')
21
22 self.sdk_target_sysroot = os.path.join(self.sdk_output, self.target_path)
23 self.sdk_host_sysroot = self.sdk_output
24
25 if manifest_dir is None:
26 self.manifest_dir = self.d.getVar("SDK_DIR")
27 else:
28 self.manifest_dir = manifest_dir
29
30 self.remove(self.sdk_output, True)
31
32 self.install_order = Manifest.INSTALL_ORDER
33
34 @abstractmethod
35 def _populate(self):
36 pass
37
38 def populate(self):
39 self.mkdirhier(self.sdk_output)
40
41 # call backend dependent implementation
42 self._populate()
43
44 # Don't ship any libGL in the SDK
45 self.remove(os.path.join(self.sdk_output, self.sdk_native_path,
46 self.d.getVar('libdir_nativesdk').strip('/'),
47 "libGL*"))
48
49 # Fix or remove broken .la files
50 self.remove(os.path.join(self.sdk_output, self.sdk_native_path,
51 self.d.getVar('libdir_nativesdk').strip('/'),
52 "*.la"))
53
54 # Link the ld.so.cache file into the hosts filesystem
55 link_name = os.path.join(self.sdk_output, self.sdk_native_path,
56 self.sysconfdir, "ld.so.cache")
57 self.mkdirhier(os.path.dirname(link_name))
58 os.symlink("/etc/ld.so.cache", link_name)
59
60 execute_pre_post_process(self.d, self.d.getVar('SDK_POSTPROCESS_COMMAND'))
61
62 def movefile(self, sourcefile, destdir):
63 try:
64 # FIXME: this check of movefile's return code to None should be
65 # fixed within the function to use only exceptions to signal when
66 # something goes wrong
67 if (bb.utils.movefile(sourcefile, destdir) == None):
68 raise OSError("moving %s to %s failed"
69 %(sourcefile, destdir))
70 #FIXME: using umbrella exc catching because bb.utils method raises it
71 except Exception as e:
72 bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc())
73 bb.fatal("unable to place %s in final SDK location" % sourcefile)
74
75 def mkdirhier(self, dirpath):
76 try:
77 bb.utils.mkdirhier(dirpath)
78 except OSError as e:
79 bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc())
80 bb.fatal("cannot make dir for SDK: %s" % dirpath)
81
82 def remove(self, path, recurse=False):
83 try:
84 bb.utils.remove(path, recurse)
85 #FIXME: using umbrella exc catching because bb.utils method raises it
86 except Exception as e:
87 bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc())
88 bb.warn("cannot remove SDK dir: %s" % path)
89
90 def install_locales(self, pm):
91 linguas = self.d.getVar("SDKIMAGE_LINGUAS")
92 if linguas:
93 import fnmatch
94 # Install the binary locales
95 if linguas == "all":
96 pm.install_glob("nativesdk-glibc-binary-localedata-*.utf-8", sdk=True)
97 else:
98 pm.install(["nativesdk-glibc-binary-localedata-%s.utf-8" % \
99 lang for lang in linguas.split()])
100 # Generate a locale archive of them
101 target_arch = self.d.getVar('SDK_ARCH')
102 rootfs = oe.path.join(self.sdk_host_sysroot, self.sdk_native_path)
103 localedir = oe.path.join(rootfs, self.d.getVar("libdir_nativesdk"), "locale")
104 generate_locale_archive(self.d, rootfs, target_arch, localedir)
105 # And now delete the binary locales
106 pkgs = fnmatch.filter(pm.list_installed(), "nativesdk-glibc-binary-localedata-*.utf-8")
107 pm.remove(pkgs)
108 else:
109 # No linguas so do nothing
110 pass
111
112
113def sdk_list_installed_packages(d, target, rootfs_dir=None):
114 if rootfs_dir is None:
115 sdk_output = d.getVar('SDK_OUTPUT')
116 target_path = d.getVar('SDKTARGETSYSROOT').strip('/')
117
118 rootfs_dir = [sdk_output, os.path.join(sdk_output, target_path)][target is True]
119
120 if target is False:
121 ipkgconf_sdk_target = d.getVar("IPKGCONF_SDK")
122 d.setVar("IPKGCONF_TARGET", ipkgconf_sdk_target)
123
124 img_type = d.getVar('IMAGE_PKGTYPE')
125 import importlib
126 cls = importlib.import_module('oe.package_manager.' + img_type)
127 return cls.PMPkgsList(d, rootfs_dir).list_pkgs()
128
129def populate_sdk(d, manifest_dir=None):
130 env_bkp = os.environ.copy()
131
132 img_type = d.getVar('IMAGE_PKGTYPE')
133 import importlib
134 cls = importlib.import_module('oe.package_manager.' + img_type + '.sdk')
135 cls.PkgSdk(d, manifest_dir).populate()
136
137 os.environ.clear()
138 os.environ.update(env_bkp)
139
140def get_extra_sdkinfo(sstate_dir):
141 """
142 This function is going to be used for generating the target and host manifest files packages of eSDK.
143 """
144 import math
145
146 extra_info = {}
147 extra_info['tasksizes'] = {}
148 extra_info['filesizes'] = {}
149 for root, _, files in os.walk(sstate_dir):
150 for fn in files:
151 if fn.endswith('.tgz'):
152 fsize = int(math.ceil(float(os.path.getsize(os.path.join(root, fn))) / 1024))
153 task = fn.rsplit(':',1)[1].split('_',1)[1].split(',')[0]
154 origtotal = extra_info['tasksizes'].get(task, 0)
155 extra_info['tasksizes'][task] = origtotal + fsize
156 extra_info['filesizes'][fn] = fsize
157 return extra_info
158
159if __name__ == "__main__":
160 pass
diff --git a/meta-xilinx-core/lib/oe/spdx.py b/meta-xilinx-core/lib/oe/spdx.py
new file mode 100644
index 00000000..7aaf2af5
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/spdx.py
@@ -0,0 +1,357 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7#
8# This library is intended to capture the JSON SPDX specification in a type
9# safe manner. It is not intended to encode any particular OE specific
10# behaviors, see the sbom.py for that.
11#
12# The documented SPDX spec document doesn't cover the JSON syntax for
13# particular configuration, which can make it hard to determine what the JSON
14# syntax should be. I've found it is actually much simpler to read the official
15# SPDX JSON schema which can be found here: https://github.com/spdx/spdx-spec
16# in schemas/spdx-schema.json
17#
18
19import hashlib
20import itertools
21import json
22
23SPDX_VERSION = "2.2"
24
25
26#
27# The following are the support classes that are used to implement SPDX object
28#
29
30class _Property(object):
31 """
32 A generic SPDX object property. The different types will derive from this
33 class
34 """
35
36 def __init__(self, *, default=None):
37 self.default = default
38
39 def setdefault(self, dest, name):
40 if self.default is not None:
41 dest.setdefault(name, self.default)
42
43
44class _String(_Property):
45 """
46 A scalar string property for an SPDX object
47 """
48
49 def __init__(self, **kwargs):
50 super().__init__(**kwargs)
51
52 def set_property(self, attrs, name):
53 def get_helper(obj):
54 return obj._spdx[name]
55
56 def set_helper(obj, value):
57 obj._spdx[name] = value
58
59 def del_helper(obj):
60 del obj._spdx[name]
61
62 attrs[name] = property(get_helper, set_helper, del_helper)
63
64 def init(self, source):
65 return source
66
67
68class _Object(_Property):
69 """
70 A scalar SPDX object property of a SPDX object
71 """
72
73 def __init__(self, cls, **kwargs):
74 super().__init__(**kwargs)
75 self.cls = cls
76
77 def set_property(self, attrs, name):
78 def get_helper(obj):
79 if not name in obj._spdx:
80 obj._spdx[name] = self.cls()
81 return obj._spdx[name]
82
83 def set_helper(obj, value):
84 obj._spdx[name] = value
85
86 def del_helper(obj):
87 del obj._spdx[name]
88
89 attrs[name] = property(get_helper, set_helper)
90
91 def init(self, source):
92 return self.cls(**source)
93
94
95class _ListProperty(_Property):
96 """
97 A list of SPDX properties
98 """
99
100 def __init__(self, prop, **kwargs):
101 super().__init__(**kwargs)
102 self.prop = prop
103
104 def set_property(self, attrs, name):
105 def get_helper(obj):
106 if not name in obj._spdx:
107 obj._spdx[name] = []
108 return obj._spdx[name]
109
110 def set_helper(obj, value):
111 obj._spdx[name] = list(value)
112
113 def del_helper(obj):
114 del obj._spdx[name]
115
116 attrs[name] = property(get_helper, set_helper, del_helper)
117
118 def init(self, source):
119 return [self.prop.init(o) for o in source]
120
121
122class _StringList(_ListProperty):
123 """
124 A list of strings as a property for an SPDX object
125 """
126
127 def __init__(self, **kwargs):
128 super().__init__(_String(), **kwargs)
129
130
131class _ObjectList(_ListProperty):
132 """
133 A list of SPDX objects as a property for an SPDX object
134 """
135
136 def __init__(self, cls, **kwargs):
137 super().__init__(_Object(cls), **kwargs)
138
139
140class MetaSPDXObject(type):
141 """
142 A metaclass that allows properties (anything derived from a _Property
143 class) to be defined for a SPDX object
144 """
145 def __new__(mcls, name, bases, attrs):
146 attrs["_properties"] = {}
147
148 for key in attrs.keys():
149 if isinstance(attrs[key], _Property):
150 prop = attrs[key]
151 attrs["_properties"][key] = prop
152 prop.set_property(attrs, key)
153
154 return super().__new__(mcls, name, bases, attrs)
155
156
157class SPDXObject(metaclass=MetaSPDXObject):
158 """
159 The base SPDX object; all SPDX spec classes must derive from this class
160 """
161 def __init__(self, **d):
162 self._spdx = {}
163
164 for name, prop in self._properties.items():
165 prop.setdefault(self._spdx, name)
166 if name in d:
167 self._spdx[name] = prop.init(d[name])
168
169 def serializer(self):
170 return self._spdx
171
172 def __setattr__(self, name, value):
173 if name in self._properties or name == "_spdx":
174 super().__setattr__(name, value)
175 return
176 raise KeyError("%r is not a valid SPDX property" % name)
177
178#
179# These are the SPDX objects implemented from the spec. The *only* properties
180# that can be added to these objects are ones directly specified in the SPDX
181# spec, however you may add helper functions to make operations easier.
182#
183# Defaults should *only* be specified if the SPDX spec says there is a certain
184# required value for a field (e.g. dataLicense), or if the field is mandatory
185# and has some sane "this field is unknown" (e.g. "NOASSERTION")
186#
187
188class SPDXAnnotation(SPDXObject):
189 annotationDate = _String()
190 annotationType = _String()
191 annotator = _String()
192 comment = _String()
193
194class SPDXChecksum(SPDXObject):
195 algorithm = _String()
196 checksumValue = _String()
197
198
199class SPDXRelationship(SPDXObject):
200 spdxElementId = _String()
201 relatedSpdxElement = _String()
202 relationshipType = _String()
203 comment = _String()
204 annotations = _ObjectList(SPDXAnnotation)
205
206
207class SPDXExternalReference(SPDXObject):
208 referenceCategory = _String()
209 referenceType = _String()
210 referenceLocator = _String()
211
212
213class SPDXPackageVerificationCode(SPDXObject):
214 packageVerificationCodeValue = _String()
215 packageVerificationCodeExcludedFiles = _StringList()
216
217
218class SPDXPackage(SPDXObject):
219 ALLOWED_CHECKSUMS = [
220 "SHA1",
221 "SHA224",
222 "SHA256",
223 "SHA384",
224 "SHA512",
225 "MD2",
226 "MD4",
227 "MD5",
228 "MD6",
229 ]
230
231 name = _String()
232 SPDXID = _String()
233 versionInfo = _String()
234 downloadLocation = _String(default="NOASSERTION")
235 supplier = _String(default="NOASSERTION")
236 homepage = _String()
237 licenseConcluded = _String(default="NOASSERTION")
238 licenseDeclared = _String(default="NOASSERTION")
239 summary = _String()
240 description = _String()
241 sourceInfo = _String()
242 copyrightText = _String(default="NOASSERTION")
243 licenseInfoFromFiles = _StringList(default=["NOASSERTION"])
244 externalRefs = _ObjectList(SPDXExternalReference)
245 packageVerificationCode = _Object(SPDXPackageVerificationCode)
246 hasFiles = _StringList()
247 packageFileName = _String()
248 annotations = _ObjectList(SPDXAnnotation)
249 checksums = _ObjectList(SPDXChecksum)
250
251
252class SPDXFile(SPDXObject):
253 SPDXID = _String()
254 fileName = _String()
255 licenseConcluded = _String(default="NOASSERTION")
256 copyrightText = _String(default="NOASSERTION")
257 licenseInfoInFiles = _StringList(default=["NOASSERTION"])
258 checksums = _ObjectList(SPDXChecksum)
259 fileTypes = _StringList()
260
261
262class SPDXCreationInfo(SPDXObject):
263 created = _String()
264 licenseListVersion = _String()
265 comment = _String()
266 creators = _StringList()
267
268
269class SPDXExternalDocumentRef(SPDXObject):
270 externalDocumentId = _String()
271 spdxDocument = _String()
272 checksum = _Object(SPDXChecksum)
273
274
275class SPDXExtractedLicensingInfo(SPDXObject):
276 name = _String()
277 comment = _String()
278 licenseId = _String()
279 extractedText = _String()
280
281
282class SPDXDocument(SPDXObject):
283 spdxVersion = _String(default="SPDX-" + SPDX_VERSION)
284 dataLicense = _String(default="CC0-1.0")
285 SPDXID = _String(default="SPDXRef-DOCUMENT")
286 name = _String()
287 documentNamespace = _String()
288 creationInfo = _Object(SPDXCreationInfo)
289 packages = _ObjectList(SPDXPackage)
290 files = _ObjectList(SPDXFile)
291 relationships = _ObjectList(SPDXRelationship)
292 externalDocumentRefs = _ObjectList(SPDXExternalDocumentRef)
293 hasExtractedLicensingInfos = _ObjectList(SPDXExtractedLicensingInfo)
294
295 def __init__(self, **d):
296 super().__init__(**d)
297
298 def to_json(self, f, *, sort_keys=False, indent=None, separators=None):
299 class Encoder(json.JSONEncoder):
300 def default(self, o):
301 if isinstance(o, SPDXObject):
302 return o.serializer()
303
304 return super().default(o)
305
306 sha1 = hashlib.sha1()
307 for chunk in Encoder(
308 sort_keys=sort_keys,
309 indent=indent,
310 separators=separators,
311 ).iterencode(self):
312 chunk = chunk.encode("utf-8")
313 f.write(chunk)
314 sha1.update(chunk)
315
316 return sha1.hexdigest()
317
318 @classmethod
319 def from_json(cls, f):
320 return cls(**json.load(f))
321
322 def add_relationship(self, _from, relationship, _to, *, comment=None, annotation=None):
323 if isinstance(_from, SPDXObject):
324 from_spdxid = _from.SPDXID
325 else:
326 from_spdxid = _from
327
328 if isinstance(_to, SPDXObject):
329 to_spdxid = _to.SPDXID
330 else:
331 to_spdxid = _to
332
333 r = SPDXRelationship(
334 spdxElementId=from_spdxid,
335 relatedSpdxElement=to_spdxid,
336 relationshipType=relationship,
337 )
338
339 if comment is not None:
340 r.comment = comment
341
342 if annotation is not None:
343 r.annotations.append(annotation)
344
345 self.relationships.append(r)
346
347 def find_by_spdxid(self, spdxid):
348 for o in itertools.chain(self.packages, self.files):
349 if o.SPDXID == spdxid:
350 return o
351 return None
352
353 def find_external_document_ref(self, namespace):
354 for r in self.externalDocumentRefs:
355 if r.spdxDocument == namespace:
356 return r
357 return None
diff --git a/meta-xilinx-core/lib/oe/sstatesig.py b/meta-xilinx-core/lib/oe/sstatesig.py
new file mode 100644
index 00000000..d818fce8
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/sstatesig.py
@@ -0,0 +1,691 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6import bb.siggen
7import bb.runqueue
8import oe
9import netrc
10
11def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCaches):
12 # Return True if we should keep the dependency, False to drop it
13 def isNative(x):
14 return x.endswith("-native")
15 def isCross(x):
16 return "-cross-" in x
17 def isNativeSDK(x):
18 return x.startswith("nativesdk-")
19 def isKernel(mc, fn):
20 inherits = " ".join(dataCaches[mc].inherits[fn])
21 return inherits.find("/module-base.bbclass") != -1 or inherits.find("/linux-kernel-base.bbclass") != -1
22 def isPackageGroup(mc, fn):
23 inherits = " ".join(dataCaches[mc].inherits[fn])
24 return "/packagegroup.bbclass" in inherits
25 def isAllArch(mc, fn):
26 inherits = " ".join(dataCaches[mc].inherits[fn])
27 return "/allarch.bbclass" in inherits
28 def isImage(mc, fn):
29 return "/image.bbclass" in " ".join(dataCaches[mc].inherits[fn])
30
31 depmc, _, deptaskname, depmcfn = bb.runqueue.split_tid_mcfn(dep)
32 mc, _ = bb.runqueue.split_mc(fn)
33
34 # We can skip the rm_work task signature to avoid running the task
35 # when we remove some tasks from the dependencie chain
36 # i.e INHERIT:remove = "create-spdx" will trigger the do_rm_work
37 if task == "do_rm_work":
38 return False
39
40 # (Almost) always include our own inter-task dependencies (unless it comes
41 # from a mcdepends). The exception is the special
42 # do_kernel_configme->do_unpack_and_patch dependency from archiver.bbclass.
43 if recipename == depname and depmc == mc:
44 if task == "do_kernel_configme" and deptaskname == "do_unpack_and_patch":
45 return False
46 return True
47
48 # Exclude well defined recipe->dependency
49 if "%s->%s" % (recipename, depname) in siggen.saferecipedeps:
50 return False
51
52 # Check for special wildcard
53 if "*->%s" % depname in siggen.saferecipedeps and recipename != depname:
54 return False
55
56 # Don't change native/cross/nativesdk recipe dependencies any further
57 if isNative(recipename) or isCross(recipename) or isNativeSDK(recipename):
58 return True
59
60 # Only target packages beyond here
61
62 # allarch packagegroups are assumed to have well behaved names which don't change between architecures/tunes
63 if isPackageGroup(mc, fn) and isAllArch(mc, fn) and not isNative(depname):
64 return False
65
66 # Exclude well defined machine specific configurations which don't change ABI
67 if depname in siggen.abisaferecipes and not isImage(mc, fn):
68 return False
69
70 # Kernel modules are well namespaced. We don't want to depend on the kernel's checksum
71 # if we're just doing an RRECOMMENDS:xxx = "kernel-module-*", not least because the checksum
72 # is machine specific.
73 # Therefore if we're not a kernel or a module recipe (inheriting the kernel classes)
74 # and we reccomend a kernel-module, we exclude the dependency.
75 if dataCaches and isKernel(depmc, depmcfn) and not isKernel(mc, fn):
76 for pkg in dataCaches[mc].runrecs[fn]:
77 if " ".join(dataCaches[mc].runrecs[fn][pkg]).find("kernel-module-") != -1:
78 return False
79
80 # Default to keep dependencies
81 return True
82
83def sstate_lockedsigs(d):
84 sigs = {}
85 types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES") or "").split()
86 for t in types:
87 siggen_lockedsigs_var = "SIGGEN_LOCKEDSIGS_%s" % t
88 lockedsigs = (d.getVar(siggen_lockedsigs_var) or "").split()
89 for ls in lockedsigs:
90 pn, task, h = ls.split(":", 2)
91 if pn not in sigs:
92 sigs[pn] = {}
93 sigs[pn][task] = [h, siggen_lockedsigs_var]
94 return sigs
95
96class SignatureGeneratorOEBasicHashMixIn(object):
97 supports_multiconfig_datacaches = True
98
99 def init_rundepcheck(self, data):
100 self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split()
101 self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split()
102 self.lockedsigs = sstate_lockedsigs(data)
103 self.lockedhashes = {}
104 self.lockedpnmap = {}
105 self.lockedhashfn = {}
106 self.machine = data.getVar("MACHINE")
107 self.mismatch_msgs = []
108 self.mismatch_number = 0
109 self.lockedsigs_msgs = ""
110 self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES") or
111 "").split()
112 self.unlockedrecipes = { k: "" for k in self.unlockedrecipes }
113 self._internal = False
114 pass
115
116 def tasks_resolved(self, virtmap, virtpnmap, dataCache):
117 # Translate virtual/xxx entries to PN values
118 newabisafe = []
119 for a in self.abisaferecipes:
120 if a in virtpnmap:
121 newabisafe.append(virtpnmap[a])
122 else:
123 newabisafe.append(a)
124 self.abisaferecipes = newabisafe
125 newsafedeps = []
126 for a in self.saferecipedeps:
127 a1, a2 = a.split("->")
128 if a1 in virtpnmap:
129 a1 = virtpnmap[a1]
130 if a2 in virtpnmap:
131 a2 = virtpnmap[a2]
132 newsafedeps.append(a1 + "->" + a2)
133 self.saferecipedeps = newsafedeps
134
135 def rundep_check(self, fn, recipename, task, dep, depname, dataCaches = None):
136 return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCaches)
137
138 def get_taskdata(self):
139 return (self.lockedpnmap, self.lockedhashfn, self.lockedhashes) + super().get_taskdata()
140
141 def set_taskdata(self, data):
142 self.lockedpnmap, self.lockedhashfn, self.lockedhashes = data[:3]
143 super().set_taskdata(data[3:])
144
145 def dump_sigs(self, dataCache, options):
146 if 'lockedsigs' in options:
147 sigfile = os.getcwd() + "/locked-sigs.inc"
148 bb.plain("Writing locked sigs to %s" % sigfile)
149 self.dump_lockedsigs(sigfile)
150 return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options)
151
152
153 def get_taskhash(self, tid, deps, dataCaches):
154 if tid in self.lockedhashes:
155 if self.lockedhashes[tid]:
156 return self.lockedhashes[tid]
157 else:
158 return super().get_taskhash(tid, deps, dataCaches)
159
160 h = super().get_taskhash(tid, deps, dataCaches)
161
162 (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
163
164 recipename = dataCaches[mc].pkg_fn[fn]
165 self.lockedpnmap[fn] = recipename
166 self.lockedhashfn[fn] = dataCaches[mc].hashfn[fn]
167
168 unlocked = False
169 if recipename in self.unlockedrecipes:
170 unlocked = True
171 else:
172 def recipename_from_dep(dep):
173 (depmc, _, _, depfn) = bb.runqueue.split_tid_mcfn(dep)
174 return dataCaches[depmc].pkg_fn[depfn]
175
176 # If any unlocked recipe is in the direct dependencies then the
177 # current recipe should be unlocked as well.
178 depnames = [ recipename_from_dep(x) for x in deps if mc == bb.runqueue.mc_from_tid(x)]
179 if any(x in y for y in depnames for x in self.unlockedrecipes):
180 self.unlockedrecipes[recipename] = ''
181 unlocked = True
182
183 if not unlocked and recipename in self.lockedsigs:
184 if task in self.lockedsigs[recipename]:
185 h_locked = self.lockedsigs[recipename][task][0]
186 var = self.lockedsigs[recipename][task][1]
187 self.lockedhashes[tid] = h_locked
188 self._internal = True
189 unihash = self.get_unihash(tid)
190 self._internal = False
191 #bb.warn("Using %s %s %s" % (recipename, task, h))
192
193 if h != h_locked and h_locked != unihash:
194 self.mismatch_number += 1
195 self.mismatch_msgs.append('The %s:%s sig is computed to be %s, but the sig is locked to %s in %s'
196 % (recipename, task, h, h_locked, var))
197
198 return h_locked
199
200 self.lockedhashes[tid] = False
201 #bb.warn("%s %s %s" % (recipename, task, h))
202 return h
203
204 def get_stampfile_hash(self, tid):
205 if tid in self.lockedhashes and self.lockedhashes[tid]:
206 return self.lockedhashes[tid]
207 return super().get_stampfile_hash(tid)
208
209 def get_cached_unihash(self, tid):
210 if tid in self.lockedhashes and self.lockedhashes[tid] and not self._internal:
211 return self.lockedhashes[tid]
212 return super().get_cached_unihash(tid)
213
214 def dump_sigtask(self, fn, task, stampbase, runtime):
215 tid = fn + ":" + task
216 if tid in self.lockedhashes and self.lockedhashes[tid]:
217 return
218 super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime)
219
220 def dump_lockedsigs(self, sigfile, taskfilter=None):
221 types = {}
222 for tid in self.runtaskdeps:
223 # Bitbake changed this to a tuple in newer versions
224 if isinstance(tid, tuple):
225 tid = tid[1]
226 if taskfilter:
227 if not tid in taskfilter:
228 continue
229 fn = bb.runqueue.fn_from_tid(tid)
230 t = self.lockedhashfn[fn].split(" ")[1].split(":")[5]
231 t = 't-' + t.replace('_', '-')
232 if t not in types:
233 types[t] = []
234 types[t].append(tid)
235
236 with open(sigfile, "w") as f:
237 l = sorted(types)
238 for t in l:
239 f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t)
240 types[t].sort()
241 sortedtid = sorted(types[t], key=lambda tid: self.lockedpnmap[bb.runqueue.fn_from_tid(tid)])
242 for tid in sortedtid:
243 (_, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
244 if tid not in self.taskhash:
245 continue
246 f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.get_unihash(tid) + " \\\n")
247 f.write(' "\n')
248 f.write('SIGGEN_LOCKEDSIGS_TYPES:%s = "%s"' % (self.machine, " ".join(l)))
249
250 def dump_siglist(self, sigfile, path_prefix_strip=None):
251 def strip_fn(fn):
252 nonlocal path_prefix_strip
253 if not path_prefix_strip:
254 return fn
255
256 fn_exp = fn.split(":")
257 if fn_exp[-1].startswith(path_prefix_strip):
258 fn_exp[-1] = fn_exp[-1][len(path_prefix_strip):]
259
260 return ":".join(fn_exp)
261
262 with open(sigfile, "w") as f:
263 tasks = []
264 for taskitem in self.taskhash:
265 (fn, task) = taskitem.rsplit(":", 1)
266 pn = self.lockedpnmap[fn]
267 tasks.append((pn, task, strip_fn(fn), self.taskhash[taskitem]))
268 for (pn, task, fn, taskhash) in sorted(tasks):
269 f.write('%s:%s %s %s\n' % (pn, task, fn, taskhash))
270
271 def checkhashes(self, sq_data, missed, found, d):
272 warn_msgs = []
273 error_msgs = []
274 sstate_missing_msgs = []
275 info_msgs = None
276
277 if self.lockedsigs:
278 if len(self.lockedsigs) > 10:
279 self.lockedsigs_msgs = "There are %s recipes with locked tasks (%s task(s) have non matching signature)" % (len(self.lockedsigs), self.mismatch_number)
280 else:
281 self.lockedsigs_msgs = "The following recipes have locked tasks:"
282 for pn in self.lockedsigs:
283 self.lockedsigs_msgs += " %s" % (pn)
284
285 for tid in sq_data['hash']:
286 if tid not in found:
287 for pn in self.lockedsigs:
288 taskname = bb.runqueue.taskname_from_tid(tid)
289 if sq_data['hash'][tid] in iter(self.lockedsigs[pn].values()):
290 if taskname == 'do_shared_workdir':
291 continue
292 sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?"
293 % (pn, taskname, sq_data['hash'][tid]))
294
295 checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK")
296 if checklevel == 'info':
297 info_msgs = self.lockedsigs_msgs
298 if checklevel == 'warn' or checklevel == 'info':
299 warn_msgs += self.mismatch_msgs
300 elif checklevel == 'error':
301 error_msgs += self.mismatch_msgs
302
303 checklevel = d.getVar("SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK")
304 if checklevel == 'warn':
305 warn_msgs += sstate_missing_msgs
306 elif checklevel == 'error':
307 error_msgs += sstate_missing_msgs
308
309 if info_msgs:
310 bb.note(info_msgs)
311 if warn_msgs:
312 bb.warn("\n".join(warn_msgs))
313 if error_msgs:
314 bb.fatal("\n".join(error_msgs))
315
316class SignatureGeneratorOEBasicHash(SignatureGeneratorOEBasicHashMixIn, bb.siggen.SignatureGeneratorBasicHash):
317 name = "OEBasicHash"
318
319class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHashMixIn, bb.siggen.SignatureGeneratorUniHashMixIn, bb.siggen.SignatureGeneratorBasicHash):
320 name = "OEEquivHash"
321
322 def init_rundepcheck(self, data):
323 super().init_rundepcheck(data)
324 self.server = data.getVar('BB_HASHSERVE')
325 if not self.server:
326 bb.fatal("OEEquivHash requires BB_HASHSERVE to be set")
327 self.method = data.getVar('SSTATE_HASHEQUIV_METHOD')
328 if not self.method:
329 bb.fatal("OEEquivHash requires SSTATE_HASHEQUIV_METHOD to be set")
330 self.max_parallel = int(data.getVar('BB_HASHSERVE_MAX_PARALLEL') or 1)
331 self.username = data.getVar("BB_HASHSERVE_USERNAME")
332 self.password = data.getVar("BB_HASHSERVE_PASSWORD")
333 if not self.username or not self.password:
334 try:
335 n = netrc.netrc()
336 auth = n.authenticators(self.server)
337 if auth is not None:
338 self.username, _, self.password = auth
339 except FileNotFoundError:
340 pass
341 except netrc.NetrcParseError as e:
342 bb.warn("Error parsing %s:%s: %s" % (e.filename, str(e.lineno), e.msg))
343
344# Insert these classes into siggen's namespace so it can see and select them
345bb.siggen.SignatureGeneratorOEBasicHash = SignatureGeneratorOEBasicHash
346bb.siggen.SignatureGeneratorOEEquivHash = SignatureGeneratorOEEquivHash
347
348
349def find_siginfo(pn, taskname, taskhashlist, d):
350 """ Find signature data files for comparison purposes """
351
352 import fnmatch
353 import glob
354
355 if not taskname:
356 # We have to derive pn and taskname
357 key = pn
358 if key.startswith("mc:"):
359 # mc:<mc>:<pn>:<task>
360 _, _, pn, taskname = key.split(':', 3)
361 else:
362 # <pn>:<task>
363 pn, taskname = key.split(':', 1)
364
365 hashfiles = {}
366
367 def get_hashval(siginfo):
368 if siginfo.endswith('.siginfo'):
369 return siginfo.rpartition(':')[2].partition('_')[0]
370 else:
371 return siginfo.rpartition('.')[2]
372
373 def get_time(fullpath):
374 return os.stat(fullpath).st_mtime
375
376 # First search in stamps dir
377 localdata = d.createCopy()
378 localdata.setVar('MULTIMACH_TARGET_SYS', '*')
379 localdata.setVar('PN', pn)
380 localdata.setVar('PV', '*')
381 localdata.setVar('PR', '*')
382 localdata.setVar('EXTENDPE', '')
383 stamp = localdata.getVar('STAMP')
384 if pn.startswith("gcc-source"):
385 # gcc-source shared workdir is a special case :(
386 stamp = localdata.expand("${STAMPS_DIR}/work-shared/gcc-${PV}-${PR}")
387
388 filespec = '%s.%s.sigdata.*' % (stamp, taskname)
389 foundall = False
390 import glob
391 bb.debug(1, "Calling glob.glob on {}".format(filespec))
392 for fullpath in glob.glob(filespec):
393 match = False
394 if taskhashlist:
395 for taskhash in taskhashlist:
396 if fullpath.endswith('.%s' % taskhash):
397 hashfiles[taskhash] = {'path':fullpath, 'sstate':False, 'time':get_time(fullpath)}
398 if len(hashfiles) == len(taskhashlist):
399 foundall = True
400 break
401 else:
402 hashval = get_hashval(fullpath)
403 hashfiles[hashval] = {'path':fullpath, 'sstate':False, 'time':get_time(fullpath)}
404
405 if not taskhashlist or (len(hashfiles) < 2 and not foundall):
406 # That didn't work, look in sstate-cache
407 hashes = taskhashlist or ['?' * 64]
408 localdata = bb.data.createCopy(d)
409 for hashval in hashes:
410 localdata.setVar('PACKAGE_ARCH', '*')
411 localdata.setVar('TARGET_VENDOR', '*')
412 localdata.setVar('TARGET_OS', '*')
413 localdata.setVar('PN', pn)
414 # gcc-source is a special case, same as with local stamps above
415 if pn.startswith("gcc-source"):
416 localdata.setVar('PN', "gcc")
417 localdata.setVar('PV', '*')
418 localdata.setVar('PR', '*')
419 localdata.setVar('BB_TASKHASH', hashval)
420 localdata.setVar('SSTATE_CURRTASK', taskname[3:])
421 swspec = localdata.getVar('SSTATE_SWSPEC')
422 if taskname in ['do_fetch', 'do_unpack', 'do_patch', 'do_populate_lic', 'do_preconfigure'] and swspec:
423 localdata.setVar('SSTATE_PKGSPEC', '${SSTATE_SWSPEC}')
424 elif pn.endswith('-native') or "-cross-" in pn or "-crosssdk-" in pn:
425 localdata.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/")
426 filespec = '%s.siginfo' % localdata.getVar('SSTATE_PKG')
427
428 bb.debug(1, "Calling glob.glob on {}".format(filespec))
429 matchedfiles = glob.glob(filespec)
430 for fullpath in matchedfiles:
431 actual_hashval = get_hashval(fullpath)
432 if actual_hashval in hashfiles:
433 continue
434 hashfiles[actual_hashval] = {'path':fullpath, 'sstate':True, 'time':get_time(fullpath)}
435
436 return hashfiles
437
438bb.siggen.find_siginfo = find_siginfo
439bb.siggen.find_siginfo_version = 2
440
441
442def sstate_get_manifest_filename(task, d):
443 """
444 Return the sstate manifest file path for a particular task.
445 Also returns the datastore that can be used to query related variables.
446 """
447 d2 = d.createCopy()
448 extrainf = d.getVarFlag("do_" + task, 'stamp-extra-info')
449 if extrainf:
450 d2.setVar("SSTATE_MANMACH", extrainf)
451 return (d2.expand("${SSTATE_MANFILEPREFIX}.%s" % task), d2)
452
453def find_sstate_manifest(taskdata, taskdata2, taskname, d, multilibcache):
454 d2 = d
455 variant = ''
456 curr_variant = ''
457 if d.getVar("BBEXTENDCURR") == "multilib":
458 curr_variant = d.getVar("BBEXTENDVARIANT")
459 if "virtclass-multilib" not in d.getVar("OVERRIDES"):
460 curr_variant = "invalid"
461 if taskdata2.startswith("virtual:multilib"):
462 variant = taskdata2.split(":")[2]
463 if curr_variant != variant:
464 if variant not in multilibcache:
465 multilibcache[variant] = oe.utils.get_multilib_datastore(variant, d)
466 d2 = multilibcache[variant]
467
468 if taskdata.endswith("-native"):
469 pkgarchs = ["${BUILD_ARCH}", "${BUILD_ARCH}_${ORIGNATIVELSBSTRING}"]
470 elif taskdata.startswith("nativesdk-"):
471 pkgarchs = ["${SDK_ARCH}_${SDK_OS}", "allarch"]
472 elif "-cross-canadian" in taskdata:
473 pkgarchs = ["${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}"]
474 elif "-cross-" in taskdata:
475 pkgarchs = ["${BUILD_ARCH}"]
476 elif "-crosssdk" in taskdata:
477 pkgarchs = ["${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"]
478 else:
479 pkgarchs = ['${MACHINE_ARCH}']
480 pkgarchs = pkgarchs + list(reversed(d2.getVar("PACKAGE_EXTRA_ARCHS").split()))
481 pkgarchs.append('allarch')
482 pkgarchs.append('${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}')
483
484 searched_manifests = []
485
486 for pkgarch in pkgarchs:
487 manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-%s-%s.%s" % (pkgarch, taskdata, taskname))
488 if os.path.exists(manifest):
489 return manifest, d2
490 searched_manifests.append(manifest)
491 bb.fatal("The sstate manifest for task '%s:%s' (multilib variant '%s') could not be found.\nThe pkgarchs considered were: %s.\nBut none of these manifests exists:\n %s"
492 % (taskdata, taskname, variant, d2.expand(", ".join(pkgarchs)),"\n ".join(searched_manifests)))
493 return None, d2
494
495def OEOuthashBasic(path, sigfile, task, d):
496 """
497 Basic output hash function
498
499 Calculates the output hash of a task by hashing all output file metadata,
500 and file contents.
501 """
502 import hashlib
503 import stat
504 import pwd
505 import grp
506 import re
507 import fnmatch
508
509 def update_hash(s):
510 s = s.encode('utf-8')
511 h.update(s)
512 if sigfile:
513 sigfile.write(s)
514
515 h = hashlib.sha256()
516 prev_dir = os.getcwd()
517 corebase = d.getVar("COREBASE")
518 tmpdir = d.getVar("TMPDIR")
519 include_owners = os.environ.get('PSEUDO_DISABLED') == '0'
520 if "package_write_" in task or task == "package_qa":
521 include_owners = False
522 include_timestamps = False
523 include_root = True
524 if task == "package":
525 include_timestamps = True
526 include_root = False
527 source_date_epoch = float(d.getVar("SOURCE_DATE_EPOCH"))
528 hash_version = d.getVar('HASHEQUIV_HASH_VERSION')
529 extra_sigdata = d.getVar("HASHEQUIV_EXTRA_SIGDATA")
530
531 filemaps = {}
532 for m in (d.getVar('SSTATE_HASHEQUIV_FILEMAP') or '').split():
533 entry = m.split(":")
534 if len(entry) != 3 or entry[0] != task:
535 continue
536 filemaps.setdefault(entry[1], [])
537 filemaps[entry[1]].append(entry[2])
538
539 try:
540 os.chdir(path)
541 basepath = os.path.normpath(path)
542
543 update_hash("OEOuthashBasic\n")
544 if hash_version:
545 update_hash(hash_version + "\n")
546
547 if extra_sigdata:
548 update_hash(extra_sigdata + "\n")
549
550 # It is only currently useful to get equivalent hashes for things that
551 # can be restored from sstate. Since the sstate object is named using
552 # SSTATE_PKGSPEC and the task name, those should be included in the
553 # output hash calculation.
554 update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC'))
555 update_hash("task=%s\n" % task)
556
557 for root, dirs, files in os.walk('.', topdown=True):
558 # Sort directories to ensure consistent ordering when recursing
559 dirs.sort()
560 files.sort()
561
562 def process(path):
563 s = os.lstat(path)
564
565 if stat.S_ISDIR(s.st_mode):
566 update_hash('d')
567 elif stat.S_ISCHR(s.st_mode):
568 update_hash('c')
569 elif stat.S_ISBLK(s.st_mode):
570 update_hash('b')
571 elif stat.S_ISSOCK(s.st_mode):
572 update_hash('s')
573 elif stat.S_ISLNK(s.st_mode):
574 update_hash('l')
575 elif stat.S_ISFIFO(s.st_mode):
576 update_hash('p')
577 else:
578 update_hash('-')
579
580 def add_perm(mask, on, off='-'):
581 if mask & s.st_mode:
582 update_hash(on)
583 else:
584 update_hash(off)
585
586 add_perm(stat.S_IRUSR, 'r')
587 add_perm(stat.S_IWUSR, 'w')
588 if stat.S_ISUID & s.st_mode:
589 add_perm(stat.S_IXUSR, 's', 'S')
590 else:
591 add_perm(stat.S_IXUSR, 'x')
592
593 if include_owners:
594 # Group/other permissions are only relevant in pseudo context
595 add_perm(stat.S_IRGRP, 'r')
596 add_perm(stat.S_IWGRP, 'w')
597 if stat.S_ISGID & s.st_mode:
598 add_perm(stat.S_IXGRP, 's', 'S')
599 else:
600 add_perm(stat.S_IXGRP, 'x')
601
602 add_perm(stat.S_IROTH, 'r')
603 add_perm(stat.S_IWOTH, 'w')
604 if stat.S_ISVTX & s.st_mode:
605 update_hash('t')
606 else:
607 add_perm(stat.S_IXOTH, 'x')
608
609 try:
610 update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name)
611 update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name)
612 except KeyError as e:
613 msg = ("KeyError: %s\nPath %s is owned by uid %d, gid %d, which doesn't match "
614 "any user/group on target. This may be due to host contamination." %
615 (e, os.path.abspath(path), s.st_uid, s.st_gid))
616 raise Exception(msg).with_traceback(e.__traceback__)
617
618 if include_timestamps:
619 # Need to clamp to SOURCE_DATE_EPOCH
620 if s.st_mtime > source_date_epoch:
621 update_hash(" %10d" % source_date_epoch)
622 else:
623 update_hash(" %10d" % s.st_mtime)
624
625 update_hash(" ")
626 if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode):
627 update_hash("%9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev))))
628 else:
629 update_hash(" " * 9)
630
631 filterfile = False
632 for entry in filemaps:
633 if fnmatch.fnmatch(path, entry):
634 filterfile = True
635
636 update_hash(" ")
637 if stat.S_ISREG(s.st_mode) and not filterfile:
638 update_hash("%10d" % s.st_size)
639 else:
640 update_hash(" " * 10)
641
642 update_hash(" ")
643 fh = hashlib.sha256()
644 if stat.S_ISREG(s.st_mode):
645 # Hash file contents
646 if filterfile:
647 # Need to ignore paths in crossscripts and postinst-useradd files.
648 with open(path, 'rb') as d:
649 chunk = d.read()
650 chunk = chunk.replace(bytes(basepath, encoding='utf8'), b'')
651 for entry in filemaps:
652 if not fnmatch.fnmatch(path, entry):
653 continue
654 for r in filemaps[entry]:
655 if r.startswith("regex-"):
656 chunk = re.sub(bytes(r[6:], encoding='utf8'), b'', chunk)
657 else:
658 chunk = chunk.replace(bytes(r, encoding='utf8'), b'')
659 fh.update(chunk)
660 else:
661 with open(path, 'rb') as d:
662 for chunk in iter(lambda: d.read(4096), b""):
663 fh.update(chunk)
664 update_hash(fh.hexdigest())
665 else:
666 update_hash(" " * len(fh.hexdigest()))
667
668 update_hash(" %s" % path)
669
670 if stat.S_ISLNK(s.st_mode):
671 update_hash(" -> %s" % os.readlink(path))
672
673 update_hash("\n")
674
675 # Process this directory and all its child files
676 if include_root or root != ".":
677 process(root)
678 for f in files:
679 if f == 'fixmepath':
680 continue
681 process(os.path.join(root, f))
682
683 for dir in dirs:
684 if os.path.islink(os.path.join(root, dir)):
685 process(os.path.join(root, dir))
686 finally:
687 os.chdir(prev_dir)
688
689 return h.hexdigest()
690
691
diff --git a/meta-xilinx-core/lib/oe/terminal.py b/meta-xilinx-core/lib/oe/terminal.py
new file mode 100644
index 00000000..4412bc14
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/terminal.py
@@ -0,0 +1,332 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6import logging
7import oe.classutils
8import shlex
9from bb.process import Popen, ExecutionError
10
11logger = logging.getLogger('BitBake.OE.Terminal')
12
13
14class UnsupportedTerminal(Exception):
15 pass
16
17class NoSupportedTerminals(Exception):
18 def __init__(self, terms):
19 self.terms = terms
20
21
22class Registry(oe.classutils.ClassRegistry):
23 command = None
24
25 def __init__(cls, name, bases, attrs):
26 super(Registry, cls).__init__(name.lower(), bases, attrs)
27
28 @property
29 def implemented(cls):
30 return bool(cls.command)
31
32
33class Terminal(Popen, metaclass=Registry):
34 def __init__(self, sh_cmd, title=None, env=None, d=None):
35 from subprocess import STDOUT
36 fmt_sh_cmd = self.format_command(sh_cmd, title)
37 try:
38 Popen.__init__(self, fmt_sh_cmd, env=env, stderr=STDOUT)
39 except OSError as exc:
40 import errno
41 if exc.errno == errno.ENOENT:
42 raise UnsupportedTerminal(self.name)
43 else:
44 raise
45
46 def format_command(self, sh_cmd, title):
47 fmt = {'title': title or 'Terminal', 'command': sh_cmd, 'cwd': os.getcwd() }
48 if isinstance(self.command, str):
49 return shlex.split(self.command.format(**fmt))
50 else:
51 return [element.format(**fmt) for element in self.command]
52
53class XTerminal(Terminal):
54 def __init__(self, sh_cmd, title=None, env=None, d=None):
55 Terminal.__init__(self, sh_cmd, title, env, d)
56 if not os.environ.get('DISPLAY'):
57 raise UnsupportedTerminal(self.name)
58
59class Gnome(XTerminal):
60 command = 'gnome-terminal -t "{title}" -- {command}'
61 priority = 2
62
63 def __init__(self, sh_cmd, title=None, env=None, d=None):
64 # Recent versions of gnome-terminal does not support non-UTF8 charset:
65 # https://bugzilla.gnome.org/show_bug.cgi?id=732127; as a workaround,
66 # clearing the LC_ALL environment variable so it uses the locale.
67 # Once fixed on the gnome-terminal project, this should be removed.
68 if os.getenv('LC_ALL'): os.putenv('LC_ALL','')
69
70 XTerminal.__init__(self, sh_cmd, title, env, d)
71
72class Mate(XTerminal):
73 command = 'mate-terminal --disable-factory -t "{title}" -x {command}'
74 priority = 2
75
76class Xfce(XTerminal):
77 command = 'xfce4-terminal -T "{title}" -e "{command}"'
78 priority = 2
79
80class Terminology(XTerminal):
81 command = 'terminology -T="{title}" -e {command}'
82 priority = 2
83
84class Konsole(XTerminal):
85 command = 'konsole --separate --workdir . -p tabtitle="{title}" -e {command}'
86 priority = 2
87
88 def __init__(self, sh_cmd, title=None, env=None, d=None):
89 # Check version
90 vernum = check_terminal_version("konsole")
91 if vernum and bb.utils.vercmp_string_op(vernum, "2.0.0", "<"):
92 # Konsole from KDE 3.x
93 self.command = 'konsole -T "{title}" -e {command}'
94 elif vernum and bb.utils.vercmp_string_op(vernum, "16.08.1", "<"):
95 # Konsole pre 16.08.01 Has nofork
96 self.command = 'konsole --nofork --workdir . -p tabtitle="{title}" -e {command}'
97 XTerminal.__init__(self, sh_cmd, title, env, d)
98
99class XTerm(XTerminal):
100 command = 'xterm -T "{title}" -e {command}'
101 priority = 1
102
103class Rxvt(XTerminal):
104 command = 'rxvt -T "{title}" -e {command}'
105 priority = 1
106
107class URxvt(XTerminal):
108 command = 'urxvt -T "{title}" -e {command}'
109 priority = 1
110
111class Screen(Terminal):
112 command = 'screen -D -m -t "{title}" -S devshell {command}'
113
114 def __init__(self, sh_cmd, title=None, env=None, d=None):
115 s_id = "devshell_%i" % os.getpid()
116 self.command = "screen -D -m -t \"{title}\" -S %s {command}" % s_id
117 Terminal.__init__(self, sh_cmd, title, env, d)
118 msg = 'Screen started. Please connect in another terminal with ' \
119 '"screen -r %s"' % s_id
120 if (d):
121 bb.event.fire(bb.event.LogExecTTY(msg, "screen -r %s" % s_id,
122 0.5, 10), d)
123 else:
124 logger.warning(msg)
125
126class TmuxRunning(Terminal):
127 """Open a new pane in the current running tmux window"""
128 name = 'tmux-running'
129 command = 'tmux split-window -c "{cwd}" "{command}"'
130 priority = 2.75
131
132 def __init__(self, sh_cmd, title=None, env=None, d=None):
133 if not bb.utils.which(os.getenv('PATH'), 'tmux'):
134 raise UnsupportedTerminal('tmux is not installed')
135
136 if not os.getenv('TMUX'):
137 raise UnsupportedTerminal('tmux is not running')
138
139 if not check_tmux_pane_size('tmux'):
140 raise UnsupportedTerminal('tmux pane too small or tmux < 1.9 version is being used')
141
142 Terminal.__init__(self, sh_cmd, title, env, d)
143
144class TmuxNewWindow(Terminal):
145 """Open a new window in the current running tmux session"""
146 name = 'tmux-new-window'
147 command = 'tmux new-window -c "{cwd}" -n "{title}" "{command}"'
148 priority = 2.70
149
150 def __init__(self, sh_cmd, title=None, env=None, d=None):
151 if not bb.utils.which(os.getenv('PATH'), 'tmux'):
152 raise UnsupportedTerminal('tmux is not installed')
153
154 if not os.getenv('TMUX'):
155 raise UnsupportedTerminal('tmux is not running')
156
157 Terminal.__init__(self, sh_cmd, title, env, d)
158
159class Tmux(Terminal):
160 """Start a new tmux session and window"""
161 command = 'tmux new -c "{cwd}" -d -s devshell -n devshell "{command}"'
162 priority = 0.75
163
164 def __init__(self, sh_cmd, title=None, env=None, d=None):
165 if not bb.utils.which(os.getenv('PATH'), 'tmux'):
166 raise UnsupportedTerminal('tmux is not installed')
167
168 # TODO: consider using a 'devshell' session shared amongst all
169 # devshells, if it's already there, add a new window to it.
170 window_name = 'devshell-%i' % os.getpid()
171
172 self.command = 'tmux new -c "{{cwd}}" -d -s {0} -n {0} "{{command}}"'
173 if not check_tmux_version('1.9'):
174 # `tmux new-session -c` was added in 1.9;
175 # older versions fail with that flag
176 self.command = 'tmux new -d -s {0} -n {0} "{{command}}"'
177 self.command = self.command.format(window_name)
178 Terminal.__init__(self, sh_cmd, title, env, d)
179
180 attach_cmd = 'tmux att -t {0}'.format(window_name)
181 msg = 'Tmux started. Please connect in another terminal with `tmux att -t {0}`'.format(window_name)
182 if d:
183 bb.event.fire(bb.event.LogExecTTY(msg, attach_cmd, 0.5, 10), d)
184 else:
185 logger.warning(msg)
186
187class Custom(Terminal):
188 command = 'false' # This is a placeholder
189 priority = 3
190
191 def __init__(self, sh_cmd, title=None, env=None, d=None):
192 self.command = d and d.getVar('OE_TERMINAL_CUSTOMCMD')
193 if self.command:
194 if not '{command}' in self.command:
195 self.command += ' {command}'
196 Terminal.__init__(self, sh_cmd, title, env, d)
197 logger.warning('Custom terminal was started.')
198 else:
199 logger.debug('No custom terminal (OE_TERMINAL_CUSTOMCMD) set')
200 raise UnsupportedTerminal('OE_TERMINAL_CUSTOMCMD not set')
201
202
203def prioritized():
204 return Registry.prioritized()
205
206def get_cmd_list():
207 terms = Registry.prioritized()
208 cmds = []
209 for term in terms:
210 if term.command:
211 cmds.append(term.command)
212 return cmds
213
214def spawn_preferred(sh_cmd, title=None, env=None, d=None):
215 """Spawn the first supported terminal, by priority"""
216 for terminal in prioritized():
217 try:
218 spawn(terminal.name, sh_cmd, title, env, d)
219 break
220 except UnsupportedTerminal:
221 pass
222 except:
223 bb.warn("Terminal %s is supported but did not start" % (terminal.name))
224 # when we've run out of options
225 else:
226 raise NoSupportedTerminals(get_cmd_list())
227
228def spawn(name, sh_cmd, title=None, env=None, d=None):
229 """Spawn the specified terminal, by name"""
230 logger.debug('Attempting to spawn terminal "%s"', name)
231 try:
232 terminal = Registry.registry[name]
233 except KeyError:
234 raise UnsupportedTerminal(name)
235
236 # We need to know when the command completes but some terminals (at least
237 # gnome and tmux) gives us no way to do this. We therefore write the pid
238 # to a file using a "phonehome" wrapper script, then monitor the pid
239 # until it exits.
240 import tempfile
241 import time
242 pidfile = tempfile.NamedTemporaryFile(delete = False).name
243 try:
244 sh_cmd = bb.utils.which(os.getenv('PATH'), "oe-gnome-terminal-phonehome") + " " + pidfile + " " + sh_cmd
245 pipe = terminal(sh_cmd, title, env, d)
246 output = pipe.communicate()[0]
247 if output:
248 output = output.decode("utf-8")
249 if pipe.returncode != 0:
250 raise ExecutionError(sh_cmd, pipe.returncode, output)
251
252 while os.stat(pidfile).st_size <= 0:
253 time.sleep(0.01)
254 continue
255 with open(pidfile, "r") as f:
256 pid = int(f.readline())
257 finally:
258 os.unlink(pidfile)
259
260 while True:
261 try:
262 os.kill(pid, 0)
263 time.sleep(0.1)
264 except OSError:
265 return
266
267def check_tmux_version(desired):
268 vernum = check_terminal_version("tmux")
269 if vernum and bb.utils.vercmp_string_op(vernum, desired, "<"):
270 return False
271 return vernum
272
273def check_tmux_pane_size(tmux):
274 import subprocess as sub
275 # On older tmux versions (<1.9), return false. The reason
276 # is that there is no easy way to get the height of the active panel
277 # on current window without nested formats (available from version 1.9)
278 if not check_tmux_version('1.9'):
279 return False
280 try:
281 p = sub.Popen('%s list-panes -F "#{?pane_active,#{pane_height},}"' % tmux,
282 shell=True,stdout=sub.PIPE,stderr=sub.PIPE)
283 out, err = p.communicate()
284 size = int(out.strip())
285 except OSError as exc:
286 import errno
287 if exc.errno == errno.ENOENT:
288 return None
289 else:
290 raise
291
292 return size/2 >= 19
293
294def check_terminal_version(terminalName):
295 import subprocess as sub
296 try:
297 cmdversion = '%s --version' % terminalName
298 if terminalName.startswith('tmux'):
299 cmdversion = '%s -V' % terminalName
300 newenv = os.environ.copy()
301 newenv["LANG"] = "C"
302 p = sub.Popen(['sh', '-c', cmdversion], stdout=sub.PIPE, stderr=sub.PIPE, env=newenv)
303 out, err = p.communicate()
304 ver_info = out.decode().rstrip().split('\n')
305 except OSError as exc:
306 import errno
307 if exc.errno == errno.ENOENT:
308 return None
309 else:
310 raise
311 vernum = None
312 for ver in ver_info:
313 if ver.startswith('Konsole'):
314 vernum = ver.split(' ')[-1]
315 if ver.startswith('GNOME Terminal'):
316 vernum = ver.split(' ')[-1]
317 if ver.startswith('MATE Terminal'):
318 vernum = ver.split(' ')[-1]
319 if ver.startswith('tmux'):
320 vernum = ver.split()[-1]
321 if ver.startswith('tmux next-'):
322 vernum = ver.split()[-1][5:]
323 return vernum
324
325def distro_name():
326 try:
327 p = Popen(['lsb_release', '-i'])
328 out, err = p.communicate()
329 distro = out.split(':')[1].strip().lower()
330 except:
331 distro = "unknown"
332 return distro
diff --git a/meta-xilinx-core/lib/oe/types.py b/meta-xilinx-core/lib/oe/types.py
new file mode 100644
index 00000000..b929afb1
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/types.py
@@ -0,0 +1,188 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import errno
8import re
9import os
10
11
12class OEList(list):
13 """OpenEmbedded 'list' type
14
15 Acts as an ordinary list, but is constructed from a string value and a
16 separator (optional), and re-joins itself when converted to a string with
17 str(). Set the variable type flag to 'list' to use this type, and the
18 'separator' flag may be specified (defaulting to whitespace)."""
19
20 name = "list"
21
22 def __init__(self, value, separator = None):
23 if value is not None:
24 list.__init__(self, value.split(separator))
25 else:
26 list.__init__(self)
27
28 if separator is None:
29 self.separator = " "
30 else:
31 self.separator = separator
32
33 def __str__(self):
34 return self.separator.join(self)
35
36def choice(value, choices):
37 """OpenEmbedded 'choice' type
38
39 Acts as a multiple choice for the user. To use this, set the variable
40 type flag to 'choice', and set the 'choices' flag to a space separated
41 list of valid values."""
42 if not isinstance(value, str):
43 raise TypeError("choice accepts a string, not '%s'" % type(value))
44
45 value = value.lower()
46 choices = choices.lower()
47 if value not in choices.split():
48 raise ValueError("Invalid choice '%s'. Valid choices: %s" %
49 (value, choices))
50 return value
51
52class NoMatch(object):
53 """Stub python regex pattern object which never matches anything"""
54 def findall(self, string, flags=0):
55 return None
56
57 def finditer(self, string, flags=0):
58 return None
59
60 def match(self, flags=0):
61 return None
62
63 def search(self, string, flags=0):
64 return None
65
66 def split(self, string, maxsplit=0):
67 return None
68
69 def sub(pattern, repl, string, count=0):
70 return None
71
72 def subn(pattern, repl, string, count=0):
73 return None
74
75NoMatch = NoMatch()
76
77def regex(value, regexflags=None):
78 """OpenEmbedded 'regex' type
79
80 Acts as a regular expression, returning the pre-compiled regular
81 expression pattern object. To use this type, set the variable type flag
82 to 'regex', and optionally, set the 'regexflags' type to a space separated
83 list of the flags to control the regular expression matching (e.g.
84 FOO[regexflags] += 'ignorecase'). See the python documentation on the
85 're' module for a list of valid flags."""
86
87 flagval = 0
88 if regexflags:
89 for flag in regexflags.split():
90 flag = flag.upper()
91 try:
92 flagval |= getattr(re, flag)
93 except AttributeError:
94 raise ValueError("Invalid regex flag '%s'" % flag)
95
96 if not value:
97 # Let's ensure that the default behavior for an undefined or empty
98 # variable is to match nothing. If the user explicitly wants to match
99 # anything, they can match '.*' instead.
100 return NoMatch
101
102 try:
103 return re.compile(value, flagval)
104 except re.error as exc:
105 raise ValueError("Invalid regex value '%s': %s" %
106 (value, exc.args[0]))
107
108def boolean(value):
109 """OpenEmbedded 'boolean' type
110
111 Valid values for true: 'yes', 'y', 'true', 't', '1'
112 Valid values for false: 'no', 'n', 'false', 'f', '0', None
113 """
114 if value is None:
115 return False
116
117 if isinstance(value, bool):
118 return value
119
120 if not isinstance(value, str):
121 raise TypeError("boolean accepts a string, not '%s'" % type(value))
122
123 value = value.lower()
124 if value in ('yes', 'y', 'true', 't', '1'):
125 return True
126 elif value in ('no', 'n', 'false', 'f', '0'):
127 return False
128 raise ValueError("Invalid boolean value '%s'" % value)
129
130def integer(value, numberbase=10):
131 """OpenEmbedded 'integer' type
132
133 Defaults to base 10, but this can be specified using the optional
134 'numberbase' flag."""
135
136 return int(value, int(numberbase))
137
138_float = float
139def float(value, fromhex='false'):
140 """OpenEmbedded floating point type
141
142 To use this type, set the type flag to 'float', and optionally set the
143 'fromhex' flag to a true value (obeying the same rules as for the
144 'boolean' type) if the value is in base 16 rather than base 10."""
145
146 if boolean(fromhex):
147 return _float.fromhex(value)
148 else:
149 return _float(value)
150
151def path(value, relativeto='', normalize='true', mustexist='false'):
152 value = os.path.join(relativeto, value)
153
154 if boolean(normalize):
155 value = os.path.normpath(value)
156
157 if boolean(mustexist):
158 try:
159 with open(value, 'r'):
160 pass
161 except IOError as exc:
162 if exc.errno == errno.ENOENT:
163 raise ValueError("{0}: {1}".format(value, os.strerror(errno.ENOENT)))
164
165 return value
166
167def is_x86(arch):
168 """
169 Check whether arch is x86 or x86_64
170 """
171 if arch.startswith('x86_') or re.match('i.*86', arch):
172 return True
173 else:
174 return False
175
176def qemu_use_kvm(kvm, target_arch):
177 """
178 Enable kvm if target_arch == build_arch or both of them are x86 archs.
179 """
180
181 use_kvm = False
182 if kvm and boolean(kvm):
183 build_arch = os.uname()[4]
184 if is_x86(build_arch) and is_x86(target_arch):
185 use_kvm = True
186 elif build_arch == target_arch:
187 use_kvm = True
188 return use_kvm
diff --git a/meta-xilinx-core/lib/oe/useradd.py b/meta-xilinx-core/lib/oe/useradd.py
new file mode 100644
index 00000000..54aa86fe
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/useradd.py
@@ -0,0 +1,71 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6import argparse
7import re
8
9class myArgumentParser(argparse.ArgumentParser):
10 def _print_message(self, message, file=None):
11 bb.warn("%s - %s: %s" % (d.getVar('PN'), pkg, message))
12
13 # This should never be called...
14 def exit(self, status=0, message=None):
15 message = message or ("%s - %s: useradd.bbclass: Argument parsing exited" % (d.getVar('PN'), pkg))
16 error(message)
17
18 def error(self, message):
19 bb.fatal(message)
20
21def split_commands(params):
22 params = re.split('''[ \t]*;[ \t]*(?=(?:[^'"]|'[^']*'|"[^"]*")*$)''', params.strip())
23 # Remove any empty items
24 return [x for x in params if x]
25
26def split_args(params):
27 params = re.split('''[ \t]+(?=(?:[^'"]|'[^']*'|"[^"]*")*$)''', params.strip())
28 # Remove any empty items
29 return [x for x in params if x]
30
31def build_useradd_parser():
32 # The following comes from --help on useradd from shadow
33 parser = myArgumentParser(prog='useradd')
34 parser.add_argument("-b", "--base-dir", metavar="BASE_DIR", help="base directory for the home directory of the new account")
35 parser.add_argument("-c", "--comment", metavar="COMMENT", help="GECOS field of the new account")
36 parser.add_argument("-d", "--home-dir", metavar="HOME_DIR", help="home directory of the new account")
37 parser.add_argument("-D", "--defaults", help="print or change default useradd configuration", action="store_true")
38 parser.add_argument("-e", "--expiredate", metavar="EXPIRE_DATE", help="expiration date of the new account")
39 parser.add_argument("-f", "--inactive", metavar="INACTIVE", help="password inactivity period of the new account")
40 parser.add_argument("-g", "--gid", metavar="GROUP", help="name or ID of the primary group of the new account")
41 parser.add_argument("-G", "--groups", metavar="GROUPS", help="list of supplementary groups of the new account")
42 parser.add_argument("-k", "--skel", metavar="SKEL_DIR", help="use this alternative skeleton directory")
43 parser.add_argument("-K", "--key", metavar="KEY=VALUE", help="override /etc/login.defs defaults")
44 parser.add_argument("-l", "--no-log-init", help="do not add the user to the lastlog and faillog databases", action="store_true")
45 parser.add_argument("-m", "--create-home", help="create the user's home directory", action="store_const", const=True)
46 parser.add_argument("-M", "--no-create-home", dest="create_home", help="do not create the user's home directory", action="store_const", const=False)
47 parser.add_argument("-N", "--no-user-group", dest="user_group", help="do not create a group with the same name as the user", action="store_const", const=False)
48 parser.add_argument("-o", "--non-unique", help="allow to create users with duplicate (non-unique UID)", action="store_true")
49 parser.add_argument("-p", "--password", metavar="PASSWORD", help="encrypted password of the new account")
50 parser.add_argument("-R", "--root", metavar="CHROOT_DIR", help="directory to chroot into")
51 parser.add_argument("-r", "--system", help="create a system account", action="store_true")
52 parser.add_argument("-s", "--shell", metavar="SHELL", help="login shell of the new account")
53 parser.add_argument("-u", "--uid", metavar="UID", help="user ID of the new account")
54 parser.add_argument("-U", "--user-group", help="create a group with the same name as the user", action="store_const", const=True)
55 parser.add_argument("LOGIN", help="Login name of the new user")
56
57 return parser
58
59def build_groupadd_parser():
60 # The following comes from --help on groupadd from shadow
61 parser = myArgumentParser(prog='groupadd')
62 parser.add_argument("-f", "--force", help="exit successfully if the group already exists, and cancel -g if the GID is already used", action="store_true")
63 parser.add_argument("-g", "--gid", metavar="GID", help="use GID for the new group")
64 parser.add_argument("-K", "--key", metavar="KEY=VALUE", help="override /etc/login.defs defaults")
65 parser.add_argument("-o", "--non-unique", help="allow to create groups with duplicate (non-unique) GID", action="store_true")
66 parser.add_argument("-p", "--password", metavar="PASSWORD", help="use this encrypted password for the new group")
67 parser.add_argument("-R", "--root", metavar="CHROOT_DIR", help="directory to chroot into")
68 parser.add_argument("-r", "--system", help="create a system account", action="store_true")
69 parser.add_argument("GROUP", help="Group name of the new group")
70
71 return parser
diff --git a/meta-xilinx-core/lib/oe/utils.py b/meta-xilinx-core/lib/oe/utils.py
new file mode 100644
index 00000000..c9c7a470
--- /dev/null
+++ b/meta-xilinx-core/lib/oe/utils.py
@@ -0,0 +1,529 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import subprocess
8import multiprocessing
9import traceback
10import errno
11
12def read_file(filename):
13 try:
14 f = open( filename, "r" )
15 except IOError as reason:
16 return "" # WARNING: can't raise an error now because of the new RDEPENDS handling. This is a bit ugly. :M:
17 else:
18 data = f.read().strip()
19 f.close()
20 return data
21 return None
22
23def ifelse(condition, iftrue = True, iffalse = False):
24 if condition:
25 return iftrue
26 else:
27 return iffalse
28
29def conditional(variable, checkvalue, truevalue, falsevalue, d):
30 if d.getVar(variable) == checkvalue:
31 return truevalue
32 else:
33 return falsevalue
34
35def vartrue(var, iftrue, iffalse, d):
36 import oe.types
37 if oe.types.boolean(d.getVar(var)):
38 return iftrue
39 else:
40 return iffalse
41
42def less_or_equal(variable, checkvalue, truevalue, falsevalue, d):
43 if float(d.getVar(variable)) <= float(checkvalue):
44 return truevalue
45 else:
46 return falsevalue
47
48def version_less_or_equal(variable, checkvalue, truevalue, falsevalue, d):
49 result = bb.utils.vercmp_string(d.getVar(variable), checkvalue)
50 if result <= 0:
51 return truevalue
52 else:
53 return falsevalue
54
55def both_contain(variable1, variable2, checkvalue, d):
56 val1 = d.getVar(variable1)
57 val2 = d.getVar(variable2)
58 val1 = set(val1.split())
59 val2 = set(val2.split())
60 if isinstance(checkvalue, str):
61 checkvalue = set(checkvalue.split())
62 else:
63 checkvalue = set(checkvalue)
64 if checkvalue.issubset(val1) and checkvalue.issubset(val2):
65 return " ".join(checkvalue)
66 else:
67 return ""
68
69def set_intersect(variable1, variable2, d):
70 """
71 Expand both variables, interpret them as lists of strings, and return the
72 intersection as a flattened string.
73
74 For example:
75 s1 = "a b c"
76 s2 = "b c d"
77 s3 = set_intersect(s1, s2)
78 => s3 = "b c"
79 """
80 val1 = set(d.getVar(variable1).split())
81 val2 = set(d.getVar(variable2).split())
82 return " ".join(val1 & val2)
83
84def prune_suffix(var, suffixes, d):
85 # See if var ends with any of the suffixes listed and
86 # remove it if found
87 for suffix in suffixes:
88 if suffix and var.endswith(suffix):
89 var = var[:-len(suffix)]
90
91 prefix = d.getVar("MLPREFIX")
92 if prefix and var.startswith(prefix):
93 var = var[len(prefix):]
94
95 return var
96
97def str_filter(f, str, d):
98 from re import match
99 return " ".join([x for x in str.split() if match(f, x, 0)])
100
101def str_filter_out(f, str, d):
102 from re import match
103 return " ".join([x for x in str.split() if not match(f, x, 0)])
104
105def build_depends_string(depends, task):
106 """Append a taskname to a string of dependencies as used by the [depends] flag"""
107 return " ".join(dep + ":" + task for dep in depends.split())
108
109def inherits(d, *classes):
110 """Return True if the metadata inherits any of the specified classes"""
111 return any(bb.data.inherits_class(cls, d) for cls in classes)
112
113def features_backfill(var,d):
114 # This construct allows the addition of new features to variable specified
115 # as var
116 # Example for var = "DISTRO_FEATURES"
117 # This construct allows the addition of new features to DISTRO_FEATURES
118 # that if not present would disable existing functionality, without
119 # disturbing distributions that have already set DISTRO_FEATURES.
120 # Distributions wanting to elide a value in DISTRO_FEATURES_BACKFILL should
121 # add the feature to DISTRO_FEATURES_BACKFILL_CONSIDERED
122 features = (d.getVar(var) or "").split()
123 backfill = (d.getVar(var+"_BACKFILL") or "").split()
124 considered = (d.getVar(var+"_BACKFILL_CONSIDERED") or "").split()
125
126 addfeatures = []
127 for feature in backfill:
128 if feature not in features and feature not in considered:
129 addfeatures.append(feature)
130
131 if addfeatures:
132 d.appendVar(var, " " + " ".join(addfeatures))
133
134def all_distro_features(d, features, truevalue="1", falsevalue=""):
135 """
136 Returns truevalue if *all* given features are set in DISTRO_FEATURES,
137 else falsevalue. The features can be given as single string or anything
138 that can be turned into a set.
139
140 This is a shorter, more flexible version of
141 bb.utils.contains("DISTRO_FEATURES", features, truevalue, falsevalue, d).
142
143 Without explicit true/false values it can be used directly where
144 Python expects a boolean:
145 if oe.utils.all_distro_features(d, "foo bar"):
146 bb.fatal("foo and bar are mutually exclusive DISTRO_FEATURES")
147
148 With just a truevalue, it can be used to include files that are meant to be
149 used only when requested via DISTRO_FEATURES:
150 require ${@ oe.utils.all_distro_features(d, "foo bar", "foo-and-bar.inc")
151 """
152 return bb.utils.contains("DISTRO_FEATURES", features, truevalue, falsevalue, d)
153
154def any_distro_features(d, features, truevalue="1", falsevalue=""):
155 """
156 Returns truevalue if at least *one* of the given features is set in DISTRO_FEATURES,
157 else falsevalue. The features can be given as single string or anything
158 that can be turned into a set.
159
160 This is a shorter, more flexible version of
161 bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d).
162
163 Without explicit true/false values it can be used directly where
164 Python expects a boolean:
165 if not oe.utils.any_distro_features(d, "foo bar"):
166 bb.fatal("foo, bar or both must be set in DISTRO_FEATURES")
167
168 With just a truevalue, it can be used to include files that are meant to be
169 used only when requested via DISTRO_FEATURES:
170 require ${@ oe.utils.any_distro_features(d, "foo bar", "foo-or-bar.inc")
171
172 """
173 return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d)
174
175def parallel_make(d, makeinst=False):
176 """
177 Return the integer value for the number of parallel threads to use when
178 building, scraped out of PARALLEL_MAKE. If no parallelization option is
179 found, returns None
180
181 e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer.
182 """
183 if makeinst:
184 pm = (d.getVar('PARALLEL_MAKEINST') or '').split()
185 else:
186 pm = (d.getVar('PARALLEL_MAKE') or '').split()
187 # look for '-j' and throw other options (e.g. '-l') away
188 while pm:
189 opt = pm.pop(0)
190 if opt == '-j':
191 v = pm.pop(0)
192 elif opt.startswith('-j'):
193 v = opt[2:].strip()
194 else:
195 continue
196
197 return int(v)
198
199 return ''
200
201def parallel_make_argument(d, fmt, limit=None, makeinst=False):
202 """
203 Helper utility to construct a parallel make argument from the number of
204 parallel threads specified in PARALLEL_MAKE.
205
206 Returns the input format string `fmt` where a single '%d' will be expanded
207 with the number of parallel threads to use. If `limit` is specified, the
208 number of parallel threads will be no larger than it. If no parallelization
209 option is found in PARALLEL_MAKE, returns an empty string
210
211 e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d") will return
212 "-n 10"
213 """
214 v = parallel_make(d, makeinst)
215 if v:
216 if limit:
217 v = min(limit, v)
218 return fmt % v
219 return ''
220
221def packages_filter_out_system(d):
222 """
223 Return a list of packages from PACKAGES with the "system" packages such as
224 PN-dbg PN-doc PN-locale-eb-gb removed.
225 """
226 pn = d.getVar('PN')
227 pkgfilter = [pn + suffix for suffix in ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev', '-src')]
228 localepkg = pn + "-locale-"
229 pkgs = []
230
231 for pkg in d.getVar('PACKAGES').split():
232 if pkg not in pkgfilter and localepkg not in pkg:
233 pkgs.append(pkg)
234 return pkgs
235
236def getstatusoutput(cmd):
237 return subprocess.getstatusoutput(cmd)
238
239
240def trim_version(version, num_parts=2):
241 """
242 Return just the first <num_parts> of <version>, split by periods. For
243 example, trim_version("1.2.3", 2) will return "1.2".
244 """
245 if type(version) is not str:
246 raise TypeError("Version should be a string")
247 if num_parts < 1:
248 raise ValueError("Cannot split to parts < 1")
249
250 parts = version.split(".")
251 trimmed = ".".join(parts[:num_parts])
252 return trimmed
253
254def cpu_count(at_least=1, at_most=64):
255 cpus = len(os.sched_getaffinity(0))
256 return max(min(cpus, at_most), at_least)
257
258def execute_pre_post_process(d, cmds):
259 if cmds is None:
260 return
261
262 cmds = cmds.replace(";", " ")
263
264 for cmd in cmds.split():
265 bb.note("Executing %s ..." % cmd)
266 bb.build.exec_func(cmd, d)
267
268def get_bb_number_threads(d):
269 return int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1)
270
271def multiprocess_launch(target, items, d, extraargs=None):
272 max_process = get_bb_number_threads(d)
273 return multiprocess_launch_mp(target, items, max_process, extraargs)
274
275# For each item in items, call the function 'target' with item as the first
276# argument, extraargs as the other arguments and handle any exceptions in the
277# parent thread
278def multiprocess_launch_mp(target, items, max_process, extraargs=None):
279
280 class ProcessLaunch(multiprocessing.Process):
281 def __init__(self, *args, **kwargs):
282 multiprocessing.Process.__init__(self, *args, **kwargs)
283 self._pconn, self._cconn = multiprocessing.Pipe()
284 self._exception = None
285 self._result = None
286
287 def run(self):
288 try:
289 ret = self._target(*self._args, **self._kwargs)
290 self._cconn.send((None, ret))
291 except Exception as e:
292 tb = traceback.format_exc()
293 self._cconn.send((e, tb))
294
295 def update(self):
296 if self._pconn.poll():
297 (e, tb) = self._pconn.recv()
298 if e is not None:
299 self._exception = (e, tb)
300 else:
301 self._result = tb
302
303 @property
304 def exception(self):
305 self.update()
306 return self._exception
307
308 @property
309 def result(self):
310 self.update()
311 return self._result
312
313 launched = []
314 errors = []
315 results = []
316 items = list(items)
317 while (items and not errors) or launched:
318 if not errors and items and len(launched) < max_process:
319 args = (items.pop(),)
320 if extraargs is not None:
321 args = args + extraargs
322 p = ProcessLaunch(target=target, args=args)
323 p.start()
324 launched.append(p)
325 for q in launched:
326 # Have to manually call update() to avoid deadlocks. The pipe can be full and
327 # transfer stalled until we try and read the results object but the subprocess won't exit
328 # as it still has data to write (https://bugs.python.org/issue8426)
329 q.update()
330 # The finished processes are joined when calling is_alive()
331 if not q.is_alive():
332 if q.exception:
333 errors.append(q.exception)
334 if q.result:
335 results.append(q.result)
336 launched.remove(q)
337 # Paranoia doesn't hurt
338 for p in launched:
339 p.join()
340 if errors:
341 msg = ""
342 for (e, tb) in errors:
343 if isinstance(e, subprocess.CalledProcessError) and e.output:
344 msg = msg + str(e) + "\n"
345 msg = msg + "Subprocess output:"
346 msg = msg + e.output.decode("utf-8", errors="ignore")
347 else:
348 msg = msg + str(e) + ": " + str(tb) + "\n"
349 bb.fatal("Fatal errors occurred in subprocesses:\n%s" % msg)
350 return results
351
352def squashspaces(string):
353 import re
354 return re.sub(r"\s+", " ", string).strip()
355
356def rprovides_map(pkgdata_dir, pkg_dict):
357 # Map file -> pkg provider
358 rprov_map = {}
359
360 for pkg in pkg_dict:
361 path_to_pkgfile = os.path.join(pkgdata_dir, 'runtime-reverse', pkg)
362 if not os.path.isfile(path_to_pkgfile):
363 continue
364 with open(path_to_pkgfile) as f:
365 for line in f:
366 if line.startswith('RPROVIDES') or line.startswith('FILERPROVIDES'):
367 # List all components provided by pkg.
368 # Exclude version strings, i.e. those starting with (
369 provides = [x for x in line.split()[1:] if not x.startswith('(')]
370 for prov in provides:
371 if prov in rprov_map:
372 rprov_map[prov].append(pkg)
373 else:
374 rprov_map[prov] = [pkg]
375
376 return rprov_map
377
378def format_pkg_list(pkg_dict, ret_format=None, pkgdata_dir=None):
379 output = []
380
381 if ret_format == "arch":
382 for pkg in sorted(pkg_dict):
383 output.append("%s %s" % (pkg, pkg_dict[pkg]["arch"]))
384 elif ret_format == "file":
385 for pkg in sorted(pkg_dict):
386 output.append("%s %s %s" % (pkg, pkg_dict[pkg]["filename"], pkg_dict[pkg]["arch"]))
387 elif ret_format == "ver":
388 for pkg in sorted(pkg_dict):
389 output.append("%s %s %s" % (pkg, pkg_dict[pkg]["arch"], pkg_dict[pkg]["ver"]))
390 elif ret_format == "deps":
391 rprov_map = rprovides_map(pkgdata_dir, pkg_dict)
392 for pkg in sorted(pkg_dict):
393 for dep in pkg_dict[pkg]["deps"]:
394 if dep in rprov_map:
395 # There could be multiple providers within the image
396 for pkg_provider in rprov_map[dep]:
397 output.append("%s|%s * %s [RPROVIDES]" % (pkg, pkg_provider, dep))
398 else:
399 output.append("%s|%s" % (pkg, dep))
400 else:
401 for pkg in sorted(pkg_dict):
402 output.append(pkg)
403
404 output_str = '\n'.join(output)
405
406 if output_str:
407 # make sure last line is newline terminated
408 output_str += '\n'
409
410 return output_str
411
412
413# Helper function to get the host compiler version
414# Do not assume the compiler is gcc
415def get_host_compiler_version(d, taskcontextonly=False):
416 import re, subprocess
417
418 if taskcontextonly and d.getVar('BB_WORKERCONTEXT') != '1':
419 return
420
421 compiler = d.getVar("BUILD_CC")
422 # Get rid of ccache since it is not present when parsing.
423 if compiler.startswith('ccache '):
424 compiler = compiler[7:]
425 try:
426 env = os.environ.copy()
427 # datastore PATH does not contain session PATH as set by environment-setup-...
428 # this breaks the install-buildtools use-case
429 # env["PATH"] = d.getVar("PATH")
430 output = subprocess.check_output("%s --version" % compiler, \
431 shell=True, env=env, stderr=subprocess.STDOUT).decode("utf-8")
432 except subprocess.CalledProcessError as e:
433 bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8")))
434
435 match = re.match(r".* (\d+\.\d+)\.\d+.*", output.split('\n')[0])
436 if not match:
437 bb.fatal("Can't get compiler version from %s --version output" % compiler)
438
439 version = match.group(1)
440 return compiler, version
441
442
443def host_gcc_version(d, taskcontextonly=False):
444 import re, subprocess
445
446 if taskcontextonly and d.getVar('BB_WORKERCONTEXT') != '1':
447 return
448
449 compiler = d.getVar("BUILD_CC")
450 # Get rid of ccache since it is not present when parsing.
451 if compiler.startswith('ccache '):
452 compiler = compiler[7:]
453 try:
454 env = os.environ.copy()
455 env["PATH"] = d.getVar("PATH")
456 output = subprocess.check_output("%s --version" % compiler, \
457 shell=True, env=env, stderr=subprocess.STDOUT).decode("utf-8")
458 except subprocess.CalledProcessError as e:
459 bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8")))
460
461 match = re.match(r".* (\d+\.\d+)\.\d+.*", output.split('\n')[0])
462 if not match:
463 bb.fatal("Can't get compiler version from %s --version output" % compiler)
464
465 version = match.group(1)
466 return "-%s" % version if version in ("4.8", "4.9") else ""
467
468
469def get_multilib_datastore(variant, d):
470 localdata = bb.data.createCopy(d)
471 if variant:
472 overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + variant
473 localdata.setVar("OVERRIDES", overrides)
474 localdata.setVar("MLPREFIX", variant + "-")
475 else:
476 origdefault = localdata.getVar("DEFAULTTUNE_MULTILIB_ORIGINAL")
477 if origdefault:
478 localdata.setVar("DEFAULTTUNE", origdefault)
479 overrides = localdata.getVar("OVERRIDES", False).split(":")
480 overrides = ":".join([x for x in overrides if not x.startswith("virtclass-multilib-")])
481 localdata.setVar("OVERRIDES", overrides)
482 localdata.setVar("MLPREFIX", "")
483 return localdata
484
485def sh_quote(string):
486 import shlex
487 return shlex.quote(string)
488
489def directory_size(root, blocksize=4096):
490 """
491 Calculate the size of the directory, taking into account hard links,
492 rounding up every size to multiples of the blocksize.
493 """
494 def roundup(size):
495 """
496 Round the size up to the nearest multiple of the block size.
497 """
498 import math
499 return math.ceil(size / blocksize) * blocksize
500
501 def getsize(filename):
502 """
503 Get the size of the filename, not following symlinks, taking into
504 account hard links.
505 """
506 stat = os.lstat(filename)
507 if stat.st_ino not in inodes:
508 inodes.add(stat.st_ino)
509 return stat.st_size
510 else:
511 return 0
512
513 inodes = set()
514 total = 0
515 for root, dirs, files in os.walk(root):
516 total += sum(roundup(getsize(os.path.join(root, name))) for name in files)
517 total += roundup(getsize(root))
518 return total
519
520# Update the mtime of a file, skip if permission/read-only issues
521def touch(filename):
522 try:
523 os.utime(filename, None)
524 except PermissionError:
525 pass
526 except OSError as e:
527 # Handle read-only file systems gracefully
528 if e.errno != errno.EROFS:
529 raise e
diff --git a/meta-xilinx-core/scripts/lib/scriptpath.py b/meta-xilinx-core/scripts/lib/scriptpath.py
new file mode 100644
index 00000000..f32326db
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/scriptpath.py
@@ -0,0 +1,32 @@
1# Path utility functions for OE python scripts
2#
3# Copyright (C) 2012-2014 Intel Corporation
4# Copyright (C) 2011 Mentor Graphics Corporation
5#
6# SPDX-License-Identifier: GPL-2.0-only
7#
8
9import sys
10import os
11import os.path
12
13def add_oe_lib_path():
14 basepath = os.path.abspath(os.path.dirname(__file__) + '/../..')
15 newpath = basepath + '/meta/lib'
16 sys.path.insert(0, newpath)
17
18def add_bitbake_lib_path():
19 basepath = os.path.abspath(os.path.dirname(__file__) + '/../..')
20 bitbakepath = None
21 if os.path.exists(basepath + '/bitbake/lib/bb'):
22 bitbakepath = basepath + '/bitbake'
23 else:
24 # look for bitbake/bin dir in PATH
25 for pth in os.environ['PATH'].split(':'):
26 if os.path.exists(os.path.join(pth, '../lib/bb')):
27 bitbakepath = os.path.abspath(os.path.join(pth, '..'))
28 break
29
30 if bitbakepath:
31 sys.path.insert(0, bitbakepath + '/lib')
32 return bitbakepath
diff --git a/meta-xilinx-core/scripts/lib/wic/__init__.py b/meta-xilinx-core/scripts/lib/wic/__init__.py
new file mode 100644
index 00000000..85567934
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/__init__.py
@@ -0,0 +1,10 @@
1#!/usr/bin/env python3
2#
3# Copyright (c) 2007 Red Hat, Inc.
4# Copyright (c) 2011 Intel, Inc.
5#
6# SPDX-License-Identifier: GPL-2.0-only
7#
8
9class WicError(Exception):
10 pass
diff --git a/meta-xilinx-core/scripts/lib/wic/canned-wks/common.wks.inc b/meta-xilinx-core/scripts/lib/wic/canned-wks/common.wks.inc
new file mode 100644
index 00000000..89880b41
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/canned-wks/common.wks.inc
@@ -0,0 +1,3 @@
1# This file is included into 3 canned wks files from this directory
2part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
3part / --source rootfs --use-uuid --fstype=ext4 --label platform --align 1024
diff --git a/meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg b/meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg
new file mode 100644
index 00000000..c58e74a8
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg
@@ -0,0 +1,27 @@
1# This is an example configuration file for syslinux.
2TIMEOUT 50
3ALLOWOPTIONS 1
4SERIAL 0 115200
5PROMPT 0
6
7UI vesamenu.c32
8menu title Select boot options
9menu tabmsg Press [Tab] to edit, [Return] to select
10
11DEFAULT Graphics console boot
12
13LABEL Graphics console boot
14KERNEL /vmlinuz
15APPEND label=boot rootwait
16
17LABEL Serial console boot
18KERNEL /vmlinuz
19APPEND label=boot rootwait console=ttyS0,115200
20
21LABEL Graphics console install
22KERNEL /vmlinuz
23APPEND label=install rootwait
24
25LABEL Serial console install
26KERNEL /vmlinuz
27APPEND label=install rootwait console=ttyS0,115200
diff --git a/meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-bootloader-config.wks b/meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-bootloader-config.wks
new file mode 100644
index 00000000..3529e05c
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-bootloader-config.wks
@@ -0,0 +1,8 @@
1# short-description: Create a 'pcbios' direct disk image with custom bootloader config
2# long-description: Creates a partitioned legacy BIOS disk image that the user
3# can directly dd to boot media. The bootloader configuration source is a user file.
4
5include common.wks.inc
6
7bootloader --configfile="directdisk-bootloader-config.cfg"
8
diff --git a/meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-gpt.wks b/meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-gpt.wks
new file mode 100644
index 00000000..8d7d8de6
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-gpt.wks
@@ -0,0 +1,10 @@
1# short-description: Create a 'pcbios' direct disk image
2# long-description: Creates a partitioned legacy BIOS disk image that the user
3# can directly dd to boot media.
4
5
6part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
7part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
8
9bootloader --ptable gpt --timeout=0 --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 console=ttyS0,115200n8"
10
diff --git a/meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks b/meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks
new file mode 100644
index 00000000..f61d941d
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks
@@ -0,0 +1,23 @@
1# short-description: Create multi rootfs image using rootfs plugin
2# long-description: Creates a partitioned disk image with two rootfs partitions
3# using rootfs plugin.
4#
5# Partitions can use either
6# - indirect rootfs references to image recipe(s):
7# wic create directdisk-multi-indirect-recipes -e core-image-minimal \
8# --rootfs-dir rootfs1=core-image-minimal
9# --rootfs-dir rootfs2=core-image-minimal-dev
10#
11# - or paths to rootfs directories:
12# wic create directdisk-multi-rootfs \
13# --rootfs-dir rootfs1=tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs/
14# --rootfs-dir rootfs2=tmp/work/qemux86_64-poky-linux/core-image-minimal-dev/1.0-r0/rootfs/
15#
16# - or any combinations of -r and --rootfs command line options
17
18part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
19part / --source rootfs --rootfs-dir=rootfs1 --ondisk sda --fstype=ext4 --label platform --align 1024
20part /rescue --source rootfs --rootfs-dir=rootfs2 --ondisk sda --fstype=ext4 --label secondary --align 1024
21
22bootloader --timeout=0 --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 console=ttyS0,115200n8"
23
diff --git a/meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk.wks b/meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk.wks
new file mode 100644
index 00000000..8c8e06b0
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/canned-wks/directdisk.wks
@@ -0,0 +1,8 @@
1# short-description: Create a 'pcbios' direct disk image
2# long-description: Creates a partitioned legacy BIOS disk image that the user
3# can directly dd to boot media.
4
5include common.wks.inc
6
7bootloader --timeout=0 --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 console=ttyS0,115200n8"
8
diff --git a/meta-xilinx-core/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in b/meta-xilinx-core/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in
new file mode 100644
index 00000000..2fd286ff
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in
@@ -0,0 +1,3 @@
1bootloader --ptable gpt
2part /boot --source rootfs --rootfs-dir=${IMAGE_ROOTFS}/boot --fstype=vfat --label boot --active --align 1024 --use-uuid --overhead-factor 1.1
3part / --source rootfs --fstype=ext4 --label root --align 1024 --exclude-path boot/
diff --git a/meta-xilinx-core/scripts/lib/wic/canned-wks/mkefidisk.wks b/meta-xilinx-core/scripts/lib/wic/canned-wks/mkefidisk.wks
new file mode 100644
index 00000000..9f534fe1
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/canned-wks/mkefidisk.wks
@@ -0,0 +1,11 @@
1# short-description: Create an EFI disk image
2# long-description: Creates a partitioned EFI disk image that the user
3# can directly dd to boot media.
4
5part /boot --source bootimg-efi --sourceparams="loader=grub-efi" --ondisk sda --label msdos --active --align 1024
6
7part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
8
9part swap --ondisk sda --size 44 --label swap1 --fstype=swap
10
11bootloader --ptable gpt --timeout=5 --append="rootfstype=ext4 console=ttyS0,115200 console=tty0"
diff --git a/meta-xilinx-core/scripts/lib/wic/canned-wks/mkhybridiso.wks b/meta-xilinx-core/scripts/lib/wic/canned-wks/mkhybridiso.wks
new file mode 100644
index 00000000..48c5ac47
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/canned-wks/mkhybridiso.wks
@@ -0,0 +1,7 @@
1# short-description: Create a hybrid ISO image
2# long-description: Creates an EFI and legacy bootable hybrid ISO image
3# which can be used on optical media as well as USB media.
4
5part /boot --source isoimage-isohybrid --sourceparams="loader=grub-efi,image_name=HYBRID_ISO_IMG" --ondisk cd --label HYBRIDISO
6
7bootloader --timeout=15 --append=""
diff --git a/meta-xilinx-core/scripts/lib/wic/canned-wks/qemuloongarch.wks b/meta-xilinx-core/scripts/lib/wic/canned-wks/qemuloongarch.wks
new file mode 100644
index 00000000..8465c7a8
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/canned-wks/qemuloongarch.wks
@@ -0,0 +1,3 @@
1# short-description: Create qcow2 image for LoongArch QEMU machines
2
3part / --source rootfs --fstype=ext4 --label root --align 4096 --size 5G
diff --git a/meta-xilinx-core/scripts/lib/wic/canned-wks/qemuriscv.wks b/meta-xilinx-core/scripts/lib/wic/canned-wks/qemuriscv.wks
new file mode 100644
index 00000000..12c68b70
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/canned-wks/qemuriscv.wks
@@ -0,0 +1,3 @@
1# short-description: Create qcow2 image for RISC-V QEMU machines
2
3part / --source rootfs --fstype=ext4 --label root --align 4096 --size 5G
diff --git a/meta-xilinx-core/scripts/lib/wic/canned-wks/qemux86-directdisk.wks b/meta-xilinx-core/scripts/lib/wic/canned-wks/qemux86-directdisk.wks
new file mode 100644
index 00000000..80899761
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/canned-wks/qemux86-directdisk.wks
@@ -0,0 +1,8 @@
1# short-description: Create a qemu machine 'pcbios' direct disk image
2# long-description: Creates a partitioned legacy BIOS disk image that the user
3# can directly use to boot a qemu machine.
4
5include common.wks.inc
6
7bootloader --timeout=0 --append="rw oprofile.timer=1 rootfstype=ext4 console=tty console=ttyS0 "
8
diff --git a/meta-xilinx-core/scripts/lib/wic/canned-wks/sdimage-bootpart.wks b/meta-xilinx-core/scripts/lib/wic/canned-wks/sdimage-bootpart.wks
new file mode 100644
index 00000000..63bc4dab
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/canned-wks/sdimage-bootpart.wks
@@ -0,0 +1,6 @@
1# short-description: Create SD card image with a boot partition
2# long-description: Creates a partitioned SD card image. Boot files
3# are located in the first vfat partition.
4
5part /boot --source bootimg-partition --ondisk mmcblk0 --fstype=vfat --label boot --active --align 4 --size 16
6part / --source rootfs --ondisk mmcblk0 --fstype=ext4 --label root --align 4
diff --git a/meta-xilinx-core/scripts/lib/wic/canned-wks/systemd-bootdisk.wks b/meta-xilinx-core/scripts/lib/wic/canned-wks/systemd-bootdisk.wks
new file mode 100644
index 00000000..95d7b97a
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/canned-wks/systemd-bootdisk.wks
@@ -0,0 +1,11 @@
1# short-description: Create an EFI disk image with systemd-boot
2# long-description: Creates a partitioned EFI disk image that the user
3# can directly dd to boot media. The selected bootloader is systemd-boot.
4
5part /boot --source bootimg-efi --sourceparams="loader=systemd-boot" --ondisk sda --label msdos --active --align 1024 --use-uuid
6
7part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
8
9part swap --ondisk sda --size 44 --label swap1 --fstype=swap --use-uuid
10
11bootloader --ptable gpt --timeout=5 --append="rootwait rootfstype=ext4 console=ttyS0,115200 console=tty0"
diff --git a/meta-xilinx-core/scripts/lib/wic/engine.py b/meta-xilinx-core/scripts/lib/wic/engine.py
new file mode 100644
index 00000000..ce7e6c5d
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/engine.py
@@ -0,0 +1,628 @@
1#
2# Copyright (c) 2013, Intel Corporation.
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6# DESCRIPTION
7
8# This module implements the image creation engine used by 'wic' to
9# create images. The engine parses through the OpenEmbedded kickstart
10# (wks) file specified and generates images that can then be directly
11# written onto media.
12#
13# AUTHORS
14# Tom Zanussi <tom.zanussi (at] linux.intel.com>
15#
16
17import logging
18import os
19import tempfile
20import json
21import subprocess
22import shutil
23import re
24
25from collections import namedtuple, OrderedDict
26
27from wic import WicError
28from wic.filemap import sparse_copy
29from wic.pluginbase import PluginMgr
30from wic.misc import get_bitbake_var, exec_cmd
31
32logger = logging.getLogger('wic')
33
34def verify_build_env():
35 """
36 Verify that the build environment is sane.
37
38 Returns True if it is, false otherwise
39 """
40 if not os.environ.get("BUILDDIR"):
41 raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)")
42
43 return True
44
45
46CANNED_IMAGE_DIR = "lib/wic/canned-wks" # relative to scripts
47SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR
48WIC_DIR = "wic"
49
50def build_canned_image_list(path):
51 layers_path = get_bitbake_var("BBLAYERS")
52 canned_wks_layer_dirs = []
53
54 if layers_path is not None:
55 for layer_path in layers_path.split():
56 for wks_path in (WIC_DIR, SCRIPTS_CANNED_IMAGE_DIR):
57 cpath = os.path.join(layer_path, wks_path)
58 if os.path.isdir(cpath):
59 canned_wks_layer_dirs.append(cpath)
60
61 cpath = os.path.join(path, CANNED_IMAGE_DIR)
62 canned_wks_layer_dirs.append(cpath)
63
64 return canned_wks_layer_dirs
65
66def find_canned_image(scripts_path, wks_file):
67 """
68 Find a .wks file with the given name in the canned files dir.
69
70 Return False if not found
71 """
72 layers_canned_wks_dir = build_canned_image_list(scripts_path)
73
74 for canned_wks_dir in layers_canned_wks_dir:
75 for root, dirs, files in os.walk(canned_wks_dir):
76 for fname in files:
77 if fname.endswith("~") or fname.endswith("#"):
78 continue
79 if ((fname.endswith(".wks") and wks_file + ".wks" == fname) or \
80 (fname.endswith(".wks.in") and wks_file + ".wks.in" == fname)):
81 fullpath = os.path.join(canned_wks_dir, fname)
82 return fullpath
83 return None
84
85
86def list_canned_images(scripts_path):
87 """
88 List the .wks files in the canned image dir, minus the extension.
89 """
90 layers_canned_wks_dir = build_canned_image_list(scripts_path)
91
92 for canned_wks_dir in layers_canned_wks_dir:
93 for root, dirs, files in os.walk(canned_wks_dir):
94 for fname in files:
95 if fname.endswith("~") or fname.endswith("#"):
96 continue
97 if fname.endswith(".wks") or fname.endswith(".wks.in"):
98 fullpath = os.path.join(canned_wks_dir, fname)
99 with open(fullpath) as wks:
100 for line in wks:
101 desc = ""
102 idx = line.find("short-description:")
103 if idx != -1:
104 desc = line[idx + len("short-description:"):].strip()
105 break
106 basename = fname.split('.')[0]
107 print(" %s\t\t%s" % (basename.ljust(30), desc))
108
109
110def list_canned_image_help(scripts_path, fullpath):
111 """
112 List the help and params in the specified canned image.
113 """
114 found = False
115 with open(fullpath) as wks:
116 for line in wks:
117 if not found:
118 idx = line.find("long-description:")
119 if idx != -1:
120 print()
121 print(line[idx + len("long-description:"):].strip())
122 found = True
123 continue
124 if not line.strip():
125 break
126 idx = line.find("#")
127 if idx != -1:
128 print(line[idx + len("#:"):].rstrip())
129 else:
130 break
131
132
133def list_source_plugins():
134 """
135 List the available source plugins i.e. plugins available for --source.
136 """
137 plugins = PluginMgr.get_plugins('source')
138
139 for plugin in plugins:
140 print(" %s" % plugin)
141
142
143def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir,
144 native_sysroot, options):
145 """
146 Create image
147
148 wks_file - user-defined OE kickstart file
149 rootfs_dir - absolute path to the build's /rootfs dir
150 bootimg_dir - absolute path to the build's boot artifacts directory
151 kernel_dir - absolute path to the build's kernel directory
152 native_sysroot - absolute path to the build's native sysroots dir
153 image_output_dir - dirname to create for image
154 options - wic command line options (debug, bmap, etc)
155
156 Normally, the values for the build artifacts values are determined
157 by 'wic -e' from the output of the 'bitbake -e' command given an
158 image name e.g. 'core-image-minimal' and a given machine set in
159 local.conf. If that's the case, the variables get the following
160 values from the output of 'bitbake -e':
161
162 rootfs_dir: IMAGE_ROOTFS
163 kernel_dir: DEPLOY_DIR_IMAGE
164 native_sysroot: STAGING_DIR_NATIVE
165
166 In the above case, bootimg_dir remains unset and the
167 plugin-specific image creation code is responsible for finding the
168 bootimg artifacts.
169
170 In the case where the values are passed in explicitly i.e 'wic -e'
171 is not used but rather the individual 'wic' options are used to
172 explicitly specify these values.
173 """
174 try:
175 oe_builddir = os.environ["BUILDDIR"]
176 except KeyError:
177 raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)")
178
179 if not os.path.exists(options.outdir):
180 os.makedirs(options.outdir)
181
182 pname = options.imager
183 plugin_class = PluginMgr.get_plugins('imager').get(pname)
184 if not plugin_class:
185 raise WicError('Unknown plugin: %s' % pname)
186
187 plugin = plugin_class(wks_file, rootfs_dir, bootimg_dir, kernel_dir,
188 native_sysroot, oe_builddir, options)
189
190 plugin.do_create()
191
192 logger.info("The image(s) were created using OE kickstart file:\n %s", wks_file)
193
194
195def wic_list(args, scripts_path):
196 """
197 Print the list of images or source plugins.
198 """
199 if args.list_type is None:
200 return False
201
202 if args.list_type == "images":
203
204 list_canned_images(scripts_path)
205 return True
206 elif args.list_type == "source-plugins":
207 list_source_plugins()
208 return True
209 elif len(args.help_for) == 1 and args.help_for[0] == 'help':
210 wks_file = args.list_type
211 fullpath = find_canned_image(scripts_path, wks_file)
212 if not fullpath:
213 raise WicError("No image named %s found, exiting. "
214 "(Use 'wic list images' to list available images, "
215 "or specify a fully-qualified OE kickstart (.wks) "
216 "filename)" % wks_file)
217
218 list_canned_image_help(scripts_path, fullpath)
219 return True
220
221 return False
222
223
224class Disk:
225 def __init__(self, imagepath, native_sysroot, fstypes=('fat', 'ext')):
226 self.imagepath = imagepath
227 self.native_sysroot = native_sysroot
228 self.fstypes = fstypes
229 self._partitions = None
230 self._partimages = {}
231 self._lsector_size = None
232 self._psector_size = None
233 self._ptable_format = None
234
235 # find parted
236 # read paths from $PATH environment variable
237 # if it fails, use hardcoded paths
238 pathlist = "/bin:/usr/bin:/usr/sbin:/sbin/"
239 try:
240 self.paths = os.environ['PATH'] + ":" + pathlist
241 except KeyError:
242 self.paths = pathlist
243
244 if native_sysroot:
245 for path in pathlist.split(':'):
246 self.paths = "%s%s:%s" % (native_sysroot, path, self.paths)
247
248 self.parted = shutil.which("parted", path=self.paths)
249 if not self.parted:
250 raise WicError("Can't find executable parted")
251
252 self.partitions = self.get_partitions()
253
254 def __del__(self):
255 for path in self._partimages.values():
256 os.unlink(path)
257
258 def get_partitions(self):
259 if self._partitions is None:
260 self._partitions = OrderedDict()
261 out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath))
262 parttype = namedtuple("Part", "pnum start end size fstype")
263 splitted = out.splitlines()
264 # skip over possible errors in exec_cmd output
265 try:
266 idx =splitted.index("BYT;")
267 except ValueError:
268 raise WicError("Error getting partition information from %s" % (self.parted))
269 lsector_size, psector_size, self._ptable_format = splitted[idx + 1].split(":")[3:6]
270 self._lsector_size = int(lsector_size)
271 self._psector_size = int(psector_size)
272 for line in splitted[idx + 2:]:
273 pnum, start, end, size, fstype = line.split(':')[:5]
274 partition = parttype(int(pnum), int(start[:-1]), int(end[:-1]),
275 int(size[:-1]), fstype)
276 self._partitions[pnum] = partition
277
278 return self._partitions
279
280 def __getattr__(self, name):
281 """Get path to the executable in a lazy way."""
282 if name in ("mdir", "mcopy", "mdel", "mdeltree", "sfdisk", "e2fsck",
283 "resize2fs", "mkswap", "mkdosfs", "debugfs","blkid"):
284 aname = "_%s" % name
285 if aname not in self.__dict__:
286 setattr(self, aname, shutil.which(name, path=self.paths))
287 if aname not in self.__dict__ or self.__dict__[aname] is None:
288 raise WicError("Can't find executable '{}'".format(name))
289 return self.__dict__[aname]
290 return self.__dict__[name]
291
292 def _get_part_image(self, pnum):
293 if pnum not in self.partitions:
294 raise WicError("Partition %s is not in the image" % pnum)
295 part = self.partitions[pnum]
296 # check if fstype is supported
297 for fstype in self.fstypes:
298 if part.fstype.startswith(fstype):
299 break
300 else:
301 raise WicError("Not supported fstype: {}".format(part.fstype))
302 if pnum not in self._partimages:
303 tmpf = tempfile.NamedTemporaryFile(prefix="wic-part")
304 dst_fname = tmpf.name
305 tmpf.close()
306 sparse_copy(self.imagepath, dst_fname, skip=part.start, length=part.size)
307 self._partimages[pnum] = dst_fname
308
309 return self._partimages[pnum]
310
311 def _put_part_image(self, pnum):
312 """Put partition image into partitioned image."""
313 sparse_copy(self._partimages[pnum], self.imagepath,
314 seek=self.partitions[pnum].start)
315
316 def dir(self, pnum, path):
317 if pnum not in self.partitions:
318 raise WicError("Partition %s is not in the image" % pnum)
319
320 if self.partitions[pnum].fstype.startswith('ext'):
321 return exec_cmd("{} {} -R 'ls -l {}'".format(self.debugfs,
322 self._get_part_image(pnum),
323 path), as_shell=True)
324 else: # fat
325 return exec_cmd("{} -i {} ::{}".format(self.mdir,
326 self._get_part_image(pnum),
327 path))
328
329 def copy(self, src, dest):
330 """Copy partition image into wic image."""
331 pnum = dest.part if isinstance(src, str) else src.part
332
333 if self.partitions[pnum].fstype.startswith('ext'):
334 if isinstance(src, str):
335 cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\
336 format(os.path.dirname(dest.path), src, os.path.basename(src),
337 self.debugfs, self._get_part_image(pnum))
338 else: # copy from wic
339 # run both dump and rdump to support both files and directory
340 cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\
341 format(os.path.dirname(src.path), src.path,
342 dest, src.path, dest, self.debugfs,
343 self._get_part_image(pnum))
344 else: # fat
345 if isinstance(src, str):
346 cmd = "{} -i {} -snop {} ::{}".format(self.mcopy,
347 self._get_part_image(pnum),
348 src, dest.path)
349 else:
350 cmd = "{} -i {} -snop ::{} {}".format(self.mcopy,
351 self._get_part_image(pnum),
352 src.path, dest)
353
354 exec_cmd(cmd, as_shell=True)
355 self._put_part_image(pnum)
356
357 def remove_ext(self, pnum, path, recursive):
358 """
359 Remove files/dirs and their contents from the partition.
360 This only applies to ext* partition.
361 """
362 abs_path = re.sub(r'\/\/+', '/', path)
363 cmd = "{} {} -wR 'rm \"{}\"'".format(self.debugfs,
364 self._get_part_image(pnum),
365 abs_path)
366 out = exec_cmd(cmd , as_shell=True)
367 for line in out.splitlines():
368 if line.startswith("rm:"):
369 if "file is a directory" in line:
370 if recursive:
371 # loop through content and delete them one by one if
372 # flaged with -r
373 subdirs = iter(self.dir(pnum, abs_path).splitlines())
374 next(subdirs)
375 for subdir in subdirs:
376 dir = subdir.split(':')[1].split(" ", 1)[1]
377 if not dir == "." and not dir == "..":
378 self.remove_ext(pnum, "%s/%s" % (abs_path, dir), recursive)
379
380 rmdir_out = exec_cmd("{} {} -wR 'rmdir \"{}\"'".format(self.debugfs,
381 self._get_part_image(pnum),
382 abs_path.rstrip('/'))
383 , as_shell=True)
384
385 for rmdir_line in rmdir_out.splitlines():
386 if "directory not empty" in rmdir_line:
387 raise WicError("Could not complete operation: \n%s \n"
388 "use -r to remove non-empty directory" % rmdir_line)
389 if rmdir_line.startswith("rmdir:"):
390 raise WicError("Could not complete operation: \n%s "
391 "\n%s" % (str(line), rmdir_line))
392
393 else:
394 raise WicError("Could not complete operation: \n%s "
395 "\nUnable to remove %s" % (str(line), abs_path))
396
397 def remove(self, pnum, path, recursive):
398 """Remove files/dirs from the partition."""
399 partimg = self._get_part_image(pnum)
400 if self.partitions[pnum].fstype.startswith('ext'):
401 self.remove_ext(pnum, path, recursive)
402
403 else: # fat
404 cmd = "{} -i {} ::{}".format(self.mdel, partimg, path)
405 try:
406 exec_cmd(cmd)
407 except WicError as err:
408 if "not found" in str(err) or "non empty" in str(err):
409 # mdel outputs 'File ... not found' or 'directory .. non empty"
410 # try to use mdeltree as path could be a directory
411 cmd = "{} -i {} ::{}".format(self.mdeltree,
412 partimg, path)
413 exec_cmd(cmd)
414 else:
415 raise err
416 self._put_part_image(pnum)
417
418 def write(self, target, expand):
419 """Write disk image to the media or file."""
420 def write_sfdisk_script(outf, parts):
421 for key, val in parts['partitiontable'].items():
422 if key in ("partitions", "device", "firstlba", "lastlba"):
423 continue
424 if key == "id":
425 key = "label-id"
426 outf.write("{}: {}\n".format(key, val))
427 outf.write("\n")
428 for part in parts['partitiontable']['partitions']:
429 line = ''
430 for name in ('attrs', 'name', 'size', 'type', 'uuid'):
431 if name == 'size' and part['type'] == 'f':
432 # don't write size for extended partition
433 continue
434 val = part.get(name)
435 if val:
436 line += '{}={}, '.format(name, val)
437 if line:
438 line = line[:-2] # strip ', '
439 if part.get('bootable'):
440 line += ' ,bootable'
441 outf.write("{}\n".format(line))
442 outf.flush()
443
444 def read_ptable(path):
445 out = exec_cmd("{} -J {}".format(self.sfdisk, path))
446 return json.loads(out)
447
448 def write_ptable(parts, target):
449 with tempfile.NamedTemporaryFile(prefix="wic-sfdisk-", mode='w') as outf:
450 write_sfdisk_script(outf, parts)
451 cmd = "{} --no-reread {} < {} ".format(self.sfdisk, target, outf.name)
452 exec_cmd(cmd, as_shell=True)
453
454 if expand is None:
455 sparse_copy(self.imagepath, target)
456 else:
457 # copy first sectors that may contain bootloader
458 sparse_copy(self.imagepath, target, length=2048 * self._lsector_size)
459
460 # copy source partition table to the target
461 parts = read_ptable(self.imagepath)
462 write_ptable(parts, target)
463
464 # get size of unpartitioned space
465 free = None
466 for line in exec_cmd("{} -F {}".format(self.sfdisk, target)).splitlines():
467 if line.startswith("Unpartitioned space ") and line.endswith("sectors"):
468 free = int(line.split()[-2])
469 # Align free space to a 2048 sector boundary. YOCTO #12840.
470 free = free - (free % 2048)
471 if free is None:
472 raise WicError("Can't get size of unpartitioned space")
473
474 # calculate expanded partitions sizes
475 sizes = {}
476 num_auto_resize = 0
477 for num, part in enumerate(parts['partitiontable']['partitions'], 1):
478 if num in expand:
479 if expand[num] != 0: # don't resize partition if size is set to 0
480 sectors = expand[num] // self._lsector_size
481 free -= sectors - part['size']
482 part['size'] = sectors
483 sizes[num] = sectors
484 elif part['type'] != 'f':
485 sizes[num] = -1
486 num_auto_resize += 1
487
488 for num, part in enumerate(parts['partitiontable']['partitions'], 1):
489 if sizes.get(num) == -1:
490 part['size'] += free // num_auto_resize
491
492 # write resized partition table to the target
493 write_ptable(parts, target)
494
495 # read resized partition table
496 parts = read_ptable(target)
497
498 # copy partitions content
499 for num, part in enumerate(parts['partitiontable']['partitions'], 1):
500 pnum = str(num)
501 fstype = self.partitions[pnum].fstype
502
503 # copy unchanged partition
504 if part['size'] == self.partitions[pnum].size // self._lsector_size:
505 logger.info("copying unchanged partition {}".format(pnum))
506 sparse_copy(self._get_part_image(pnum), target, seek=part['start'] * self._lsector_size)
507 continue
508
509 # resize or re-create partitions
510 if fstype.startswith('ext') or fstype.startswith('fat') or \
511 fstype.startswith('linux-swap'):
512
513 partfname = None
514 with tempfile.NamedTemporaryFile(prefix="wic-part{}-".format(pnum)) as partf:
515 partfname = partf.name
516
517 if fstype.startswith('ext'):
518 logger.info("resizing ext partition {}".format(pnum))
519 partimg = self._get_part_image(pnum)
520 sparse_copy(partimg, partfname)
521 exec_cmd("{} -pf {}".format(self.e2fsck, partfname))
522 exec_cmd("{} {} {}s".format(\
523 self.resize2fs, partfname, part['size']))
524 elif fstype.startswith('fat'):
525 logger.info("copying content of the fat partition {}".format(pnum))
526 with tempfile.TemporaryDirectory(prefix='wic-fatdir-') as tmpdir:
527 # copy content to the temporary directory
528 cmd = "{} -snompi {} :: {}".format(self.mcopy,
529 self._get_part_image(pnum),
530 tmpdir)
531 exec_cmd(cmd)
532 # create new msdos partition
533 label = part.get("name")
534 label_str = "-n {}".format(label) if label else ''
535
536 cmd = "{} {} -C {} {}".format(self.mkdosfs, label_str, partfname,
537 part['size'])
538 exec_cmd(cmd)
539 # copy content from the temporary directory to the new partition
540 cmd = "{} -snompi {} {}/* ::".format(self.mcopy, partfname, tmpdir)
541 exec_cmd(cmd, as_shell=True)
542 elif fstype.startswith('linux-swap'):
543 logger.info("creating swap partition {}".format(pnum))
544 label = part.get("name")
545 label_str = "-L {}".format(label) if label else ''
546 out = exec_cmd("{} --probe {}".format(self.blkid, self._get_part_image(pnum)))
547 uuid = out[out.index("UUID=\"")+6:out.index("UUID=\"")+42]
548 uuid_str = "-U {}".format(uuid) if uuid else ''
549 with open(partfname, 'w') as sparse:
550 os.ftruncate(sparse.fileno(), part['size'] * self._lsector_size)
551 exec_cmd("{} {} {} {}".format(self.mkswap, label_str, uuid_str, partfname))
552 sparse_copy(partfname, target, seek=part['start'] * self._lsector_size)
553 os.unlink(partfname)
554 elif part['type'] != 'f':
555 logger.warning("skipping partition {}: unsupported fstype {}".format(pnum, fstype))
556
557def wic_ls(args, native_sysroot):
558 """List contents of partitioned image or vfat partition."""
559 disk = Disk(args.path.image, native_sysroot)
560 if not args.path.part:
561 if disk.partitions:
562 print('Num Start End Size Fstype')
563 for part in disk.partitions.values():
564 print("{:2d} {:12d} {:12d} {:12d} {}".format(\
565 part.pnum, part.start, part.end,
566 part.size, part.fstype))
567 else:
568 path = args.path.path or '/'
569 print(disk.dir(args.path.part, path))
570
571def wic_cp(args, native_sysroot):
572 """
573 Copy file or directory to/from the vfat/ext partition of
574 partitioned image.
575 """
576 if isinstance(args.dest, str):
577 disk = Disk(args.src.image, native_sysroot)
578 else:
579 disk = Disk(args.dest.image, native_sysroot)
580 disk.copy(args.src, args.dest)
581
582
583def wic_rm(args, native_sysroot):
584 """
585 Remove files or directories from the vfat partition of
586 partitioned image.
587 """
588 disk = Disk(args.path.image, native_sysroot)
589 disk.remove(args.path.part, args.path.path, args.recursive_delete)
590
591def wic_write(args, native_sysroot):
592 """
593 Write image to a target device.
594 """
595 disk = Disk(args.image, native_sysroot, ('fat', 'ext', 'linux-swap'))
596 disk.write(args.target, args.expand)
597
598def find_canned(scripts_path, file_name):
599 """
600 Find a file either by its path or by name in the canned files dir.
601
602 Return None if not found
603 """
604 if os.path.exists(file_name):
605 return file_name
606
607 layers_canned_wks_dir = build_canned_image_list(scripts_path)
608 for canned_wks_dir in layers_canned_wks_dir:
609 for root, dirs, files in os.walk(canned_wks_dir):
610 for fname in files:
611 if fname == file_name:
612 fullpath = os.path.join(canned_wks_dir, fname)
613 return fullpath
614
615def get_custom_config(boot_file):
616 """
617 Get the custom configuration to be used for the bootloader.
618
619 Return None if the file can't be found.
620 """
621 # Get the scripts path of poky
622 scripts_path = os.path.abspath("%s/../.." % os.path.dirname(__file__))
623
624 cfg_file = find_canned(scripts_path, boot_file)
625 if cfg_file:
626 with open(cfg_file, "r") as f:
627 config = f.read()
628 return config
diff --git a/meta-xilinx-core/scripts/lib/wic/filemap.py b/meta-xilinx-core/scripts/lib/wic/filemap.py
new file mode 100644
index 00000000..85b39d5d
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/filemap.py
@@ -0,0 +1,583 @@
1#
2# Copyright (c) 2012 Intel, Inc.
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7"""
8This module implements python implements a way to get file block. Two methods
9are supported - the FIEMAP ioctl and the 'SEEK_HOLE / SEEK_DATA' features of
10the file seek syscall. The former is implemented by the 'FilemapFiemap' class,
11the latter is implemented by the 'FilemapSeek' class. Both classes provide the
12same API. The 'filemap' function automatically selects which class can be used
13and returns an instance of the class.
14"""
15
16# Disable the following pylint recommendations:
17# * Too many instance attributes (R0902)
18# pylint: disable=R0902
19
20import errno
21import os
22import struct
23import array
24import fcntl
25import tempfile
26import logging
27
28def get_block_size(file_obj):
29 """
30 Returns block size for file object 'file_obj'. Errors are indicated by the
31 'IOError' exception.
32 """
33 # Get the block size of the host file-system for the image file by calling
34 # the FIGETBSZ ioctl (number 2).
35 try:
36 binary_data = fcntl.ioctl(file_obj, 2, struct.pack('I', 0))
37 bsize = struct.unpack('I', binary_data)[0]
38 except OSError:
39 bsize = None
40
41 # If ioctl causes OSError or give bsize to zero failback to os.fstat
42 if not bsize:
43 import os
44 stat = os.fstat(file_obj.fileno())
45 if hasattr(stat, 'st_blksize'):
46 bsize = stat.st_blksize
47 else:
48 raise IOError("Unable to determine block size")
49
50 # The logic in this script only supports a maximum of a 4KB
51 # block size
52 max_block_size = 4 * 1024
53 if bsize > max_block_size:
54 bsize = max_block_size
55
56 return bsize
57
58class ErrorNotSupp(Exception):
59 """
60 An exception of this type is raised when the 'FIEMAP' or 'SEEK_HOLE' feature
61 is not supported either by the kernel or the file-system.
62 """
63 pass
64
65class Error(Exception):
66 """A class for all the other exceptions raised by this module."""
67 pass
68
69
70class _FilemapBase(object):
71 """
72 This is a base class for a couple of other classes in this module. This
73 class simply performs the common parts of the initialization process: opens
74 the image file, gets its size, etc. The 'log' parameter is the logger object
75 to use for printing messages.
76 """
77
78 def __init__(self, image, log=None):
79 """
80 Initialize a class instance. The 'image' argument is full path to the
81 file or file object to operate on.
82 """
83
84 self._log = log
85 if self._log is None:
86 self._log = logging.getLogger(__name__)
87
88 self._f_image_needs_close = False
89
90 if hasattr(image, "fileno"):
91 self._f_image = image
92 self._image_path = image.name
93 else:
94 self._image_path = image
95 self._open_image_file()
96
97 try:
98 self.image_size = os.fstat(self._f_image.fileno()).st_size
99 except IOError as err:
100 raise Error("cannot get information about file '%s': %s"
101 % (self._f_image.name, err))
102
103 try:
104 self.block_size = get_block_size(self._f_image)
105 except IOError as err:
106 raise Error("cannot get block size for '%s': %s"
107 % (self._image_path, err))
108
109 self.blocks_cnt = self.image_size + self.block_size - 1
110 self.blocks_cnt //= self.block_size
111
112 try:
113 self._f_image.flush()
114 except IOError as err:
115 raise Error("cannot flush image file '%s': %s"
116 % (self._image_path, err))
117
118 try:
119 os.fsync(self._f_image.fileno()),
120 except OSError as err:
121 raise Error("cannot synchronize image file '%s': %s "
122 % (self._image_path, err.strerror))
123
124 self._log.debug("opened image \"%s\"" % self._image_path)
125 self._log.debug("block size %d, blocks count %d, image size %d"
126 % (self.block_size, self.blocks_cnt, self.image_size))
127
128 def __del__(self):
129 """The class destructor which just closes the image file."""
130 if self._f_image_needs_close:
131 self._f_image.close()
132
133 def _open_image_file(self):
134 """Open the image file."""
135 try:
136 self._f_image = open(self._image_path, 'rb')
137 except IOError as err:
138 raise Error("cannot open image file '%s': %s"
139 % (self._image_path, err))
140
141 self._f_image_needs_close = True
142
143 def block_is_mapped(self, block): # pylint: disable=W0613,R0201
144 """
145 This method has has to be implemented by child classes. It returns
146 'True' if block number 'block' of the image file is mapped and 'False'
147 otherwise.
148 """
149
150 raise Error("the method is not implemented")
151
152 def get_mapped_ranges(self, start, count): # pylint: disable=W0613,R0201
153 """
154 This method has has to be implemented by child classes. This is a
155 generator which yields ranges of mapped blocks in the file. The ranges
156 are tuples of 2 elements: [first, last], where 'first' is the first
157 mapped block and 'last' is the last mapped block.
158
159 The ranges are yielded for the area of the file of size 'count' blocks,
160 starting from block 'start'.
161 """
162
163 raise Error("the method is not implemented")
164
165
166# The 'SEEK_HOLE' and 'SEEK_DATA' options of the file seek system call
167_SEEK_DATA = 3
168_SEEK_HOLE = 4
169
170def _lseek(file_obj, offset, whence):
171 """This is a helper function which invokes 'os.lseek' for file object
172 'file_obj' and with specified 'offset' and 'whence'. The 'whence'
173 argument is supposed to be either '_SEEK_DATA' or '_SEEK_HOLE'. When
174 there is no more data or hole starting from 'offset', this function
175 returns '-1'. Otherwise the data or hole position is returned."""
176
177 try:
178 return os.lseek(file_obj.fileno(), offset, whence)
179 except OSError as err:
180 # The 'lseek' system call returns the ENXIO if there is no data or
181 # hole starting from the specified offset.
182 if err.errno == errno.ENXIO:
183 return -1
184 elif err.errno == errno.EINVAL:
185 raise ErrorNotSupp("the kernel or file-system does not support "
186 "\"SEEK_HOLE\" and \"SEEK_DATA\"")
187 else:
188 raise
189
190class FilemapSeek(_FilemapBase):
191 """
192 This class uses the 'SEEK_HOLE' and 'SEEK_DATA' to find file block mapping.
193 Unfortunately, the current implementation requires the caller to have write
194 access to the image file.
195 """
196
197 def __init__(self, image, log=None):
198 """Refer the '_FilemapBase' class for the documentation."""
199
200 # Call the base class constructor first
201 _FilemapBase.__init__(self, image, log)
202 self._log.debug("FilemapSeek: initializing")
203
204 self._probe_seek_hole()
205
206 def _probe_seek_hole(self):
207 """
208 Check whether the system implements 'SEEK_HOLE' and 'SEEK_DATA'.
209 Unfortunately, there seems to be no clean way for detecting this,
210 because often the system just fakes them by just assuming that all
211 files are fully mapped, so 'SEEK_HOLE' always returns EOF and
212 'SEEK_DATA' always returns the requested offset.
213
214 I could not invent a better way of detecting the fake 'SEEK_HOLE'
215 implementation than just to create a temporary file in the same
216 directory where the image file resides. It would be nice to change this
217 to something better.
218 """
219
220 directory = os.path.dirname(self._image_path)
221
222 try:
223 tmp_obj = tempfile.TemporaryFile("w+", dir=directory)
224 except IOError as err:
225 raise ErrorNotSupp("cannot create a temporary in \"%s\": %s" \
226 % (directory, err))
227
228 try:
229 os.ftruncate(tmp_obj.fileno(), self.block_size)
230 except OSError as err:
231 raise ErrorNotSupp("cannot truncate temporary file in \"%s\": %s"
232 % (directory, err))
233
234 offs = _lseek(tmp_obj, 0, _SEEK_HOLE)
235 if offs != 0:
236 # We are dealing with the stub 'SEEK_HOLE' implementation which
237 # always returns EOF.
238 self._log.debug("lseek(0, SEEK_HOLE) returned %d" % offs)
239 raise ErrorNotSupp("the file-system does not support "
240 "\"SEEK_HOLE\" and \"SEEK_DATA\" but only "
241 "provides a stub implementation")
242
243 tmp_obj.close()
244
245 def block_is_mapped(self, block):
246 """Refer the '_FilemapBase' class for the documentation."""
247 offs = _lseek(self._f_image, block * self.block_size, _SEEK_DATA)
248 if offs == -1:
249 result = False
250 else:
251 result = (offs // self.block_size == block)
252
253 self._log.debug("FilemapSeek: block_is_mapped(%d) returns %s"
254 % (block, result))
255 return result
256
257 def _get_ranges(self, start, count, whence1, whence2):
258 """
259 This function implements 'get_mapped_ranges()' depending
260 on what is passed in the 'whence1' and 'whence2' arguments.
261 """
262
263 assert whence1 != whence2
264 end = start * self.block_size
265 limit = end + count * self.block_size
266
267 while True:
268 start = _lseek(self._f_image, end, whence1)
269 if start == -1 or start >= limit or start == self.image_size:
270 break
271
272 end = _lseek(self._f_image, start, whence2)
273 if end == -1 or end == self.image_size:
274 end = self.blocks_cnt * self.block_size
275 if end > limit:
276 end = limit
277
278 start_blk = start // self.block_size
279 end_blk = end // self.block_size - 1
280 self._log.debug("FilemapSeek: yielding range (%d, %d)"
281 % (start_blk, end_blk))
282 yield (start_blk, end_blk)
283
284 def get_mapped_ranges(self, start, count):
285 """Refer the '_FilemapBase' class for the documentation."""
286 self._log.debug("FilemapSeek: get_mapped_ranges(%d, %d(%d))"
287 % (start, count, start + count - 1))
288 return self._get_ranges(start, count, _SEEK_DATA, _SEEK_HOLE)
289
290
291# Below goes the FIEMAP ioctl implementation, which is not very readable
292# because it deals with the rather complex FIEMAP ioctl. To understand the
293# code, you need to know the FIEMAP interface, which is documented in the
294# "Documentation/filesystems/fiemap.txt" file in the Linux kernel sources.
295
296# Format string for 'struct fiemap'
297_FIEMAP_FORMAT = "=QQLLLL"
298# sizeof(struct fiemap)
299_FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT)
300# Format string for 'struct fiemap_extent'
301_FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL"
302# sizeof(struct fiemap_extent)
303_FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT)
304# The FIEMAP ioctl number
305_FIEMAP_IOCTL = 0xC020660B
306# This FIEMAP ioctl flag which instructs the kernel to sync the file before
307# reading the block map
308_FIEMAP_FLAG_SYNC = 0x00000001
309# Size of the buffer for 'struct fiemap_extent' elements which will be used
310# when invoking the FIEMAP ioctl. The larger is the buffer, the less times the
311# FIEMAP ioctl will be invoked.
312_FIEMAP_BUFFER_SIZE = 256 * 1024
313
314class FilemapFiemap(_FilemapBase):
315 """
316 This class provides API to the FIEMAP ioctl. Namely, it allows to iterate
317 over all mapped blocks and over all holes.
318
319 This class synchronizes the image file every time it invokes the FIEMAP
320 ioctl in order to work-around early FIEMAP implementation kernel bugs.
321 """
322
323 def __init__(self, image, log=None):
324 """
325 Initialize a class instance. The 'image' argument is full the file
326 object to operate on.
327 """
328
329 # Call the base class constructor first
330 _FilemapBase.__init__(self, image, log)
331 self._log.debug("FilemapFiemap: initializing")
332
333 self._buf_size = _FIEMAP_BUFFER_SIZE
334
335 # Calculate how many 'struct fiemap_extent' elements fit the buffer
336 self._buf_size -= _FIEMAP_SIZE
337 self._fiemap_extent_cnt = self._buf_size // _FIEMAP_EXTENT_SIZE
338 assert self._fiemap_extent_cnt > 0
339 self._buf_size = self._fiemap_extent_cnt * _FIEMAP_EXTENT_SIZE
340 self._buf_size += _FIEMAP_SIZE
341
342 # Allocate a mutable buffer for the FIEMAP ioctl
343 self._buf = array.array('B', [0] * self._buf_size)
344
345 # Check if the FIEMAP ioctl is supported
346 self.block_is_mapped(0)
347
348 def _invoke_fiemap(self, block, count):
349 """
350 Invoke the FIEMAP ioctl for 'count' blocks of the file starting from
351 block number 'block'.
352
353 The full result of the operation is stored in 'self._buf' on exit.
354 Returns the unpacked 'struct fiemap' data structure in form of a python
355 list (just like 'struct.upack()').
356 """
357
358 if self.blocks_cnt != 0 and (block < 0 or block >= self.blocks_cnt):
359 raise Error("bad block number %d, should be within [0, %d]"
360 % (block, self.blocks_cnt))
361
362 # Initialize the 'struct fiemap' part of the buffer. We use the
363 # '_FIEMAP_FLAG_SYNC' flag in order to make sure the file is
364 # synchronized. The reason for this is that early FIEMAP
365 # implementations had many bugs related to cached dirty data, and
366 # synchronizing the file is a necessary work-around.
367 struct.pack_into(_FIEMAP_FORMAT, self._buf, 0, block * self.block_size,
368 count * self.block_size, _FIEMAP_FLAG_SYNC, 0,
369 self._fiemap_extent_cnt, 0)
370
371 try:
372 fcntl.ioctl(self._f_image, _FIEMAP_IOCTL, self._buf, 1)
373 except IOError as err:
374 # Note, the FIEMAP ioctl is supported by the Linux kernel starting
375 # from version 2.6.28 (year 2008).
376 if err.errno == errno.EOPNOTSUPP:
377 errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \
378 "by the file-system"
379 self._log.debug(errstr)
380 raise ErrorNotSupp(errstr)
381 if err.errno == errno.ENOTTY:
382 errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \
383 "by the kernel"
384 self._log.debug(errstr)
385 raise ErrorNotSupp(errstr)
386 raise Error("the FIEMAP ioctl failed for '%s': %s"
387 % (self._image_path, err))
388
389 return struct.unpack(_FIEMAP_FORMAT, self._buf[:_FIEMAP_SIZE])
390
391 def block_is_mapped(self, block):
392 """Refer the '_FilemapBase' class for the documentation."""
393 struct_fiemap = self._invoke_fiemap(block, 1)
394
395 # The 3rd element of 'struct_fiemap' is the 'fm_mapped_extents' field.
396 # If it contains zero, the block is not mapped, otherwise it is
397 # mapped.
398 result = bool(struct_fiemap[3])
399 self._log.debug("FilemapFiemap: block_is_mapped(%d) returns %s"
400 % (block, result))
401 return result
402
403 def _unpack_fiemap_extent(self, index):
404 """
405 Unpack a 'struct fiemap_extent' structure object number 'index' from
406 the internal 'self._buf' buffer.
407 """
408
409 offset = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE * index
410 return struct.unpack(_FIEMAP_EXTENT_FORMAT,
411 self._buf[offset : offset + _FIEMAP_EXTENT_SIZE])
412
413 def _do_get_mapped_ranges(self, start, count):
414 """
415 Implements most the functionality for the 'get_mapped_ranges()'
416 generator: invokes the FIEMAP ioctl, walks through the mapped extents
417 and yields mapped block ranges. However, the ranges may be consecutive
418 (e.g., (1, 100), (100, 200)) and 'get_mapped_ranges()' simply merges
419 them.
420 """
421
422 block = start
423 while block < start + count:
424 struct_fiemap = self._invoke_fiemap(block, count)
425
426 mapped_extents = struct_fiemap[3]
427 if mapped_extents == 0:
428 # No more mapped blocks
429 return
430
431 extent = 0
432 while extent < mapped_extents:
433 fiemap_extent = self._unpack_fiemap_extent(extent)
434
435 # Start of the extent
436 extent_start = fiemap_extent[0]
437 # Starting block number of the extent
438 extent_block = extent_start // self.block_size
439 # Length of the extent
440 extent_len = fiemap_extent[2]
441 # Count of blocks in the extent
442 extent_count = extent_len // self.block_size
443
444 # Extent length and offset have to be block-aligned
445 assert extent_start % self.block_size == 0
446 assert extent_len % self.block_size == 0
447
448 if extent_block > start + count - 1:
449 return
450
451 first = max(extent_block, block)
452 last = min(extent_block + extent_count, start + count) - 1
453 yield (first, last)
454
455 extent += 1
456
457 block = extent_block + extent_count
458
459 def get_mapped_ranges(self, start, count):
460 """Refer the '_FilemapBase' class for the documentation."""
461 self._log.debug("FilemapFiemap: get_mapped_ranges(%d, %d(%d))"
462 % (start, count, start + count - 1))
463 iterator = self._do_get_mapped_ranges(start, count)
464 first_prev, last_prev = next(iterator)
465
466 for first, last in iterator:
467 if last_prev == first - 1:
468 last_prev = last
469 else:
470 self._log.debug("FilemapFiemap: yielding range (%d, %d)"
471 % (first_prev, last_prev))
472 yield (first_prev, last_prev)
473 first_prev, last_prev = first, last
474
475 self._log.debug("FilemapFiemap: yielding range (%d, %d)"
476 % (first_prev, last_prev))
477 yield (first_prev, last_prev)
478
479class FilemapNobmap(_FilemapBase):
480 """
481 This class is used when both the 'SEEK_DATA/HOLE' and FIEMAP are not
482 supported by the filesystem or kernel.
483 """
484
485 def __init__(self, image, log=None):
486 """Refer the '_FilemapBase' class for the documentation."""
487
488 # Call the base class constructor first
489 _FilemapBase.__init__(self, image, log)
490 self._log.debug("FilemapNobmap: initializing")
491
492 def block_is_mapped(self, block):
493 """Refer the '_FilemapBase' class for the documentation."""
494 return True
495
496 def get_mapped_ranges(self, start, count):
497 """Refer the '_FilemapBase' class for the documentation."""
498 self._log.debug("FilemapNobmap: get_mapped_ranges(%d, %d(%d))"
499 % (start, count, start + count - 1))
500 yield (start, start + count -1)
501
502def filemap(image, log=None):
503 """
504 Create and return an instance of a Filemap class - 'FilemapFiemap' or
505 'FilemapSeek', depending on what the system we run on supports. If the
506 FIEMAP ioctl is supported, an instance of the 'FilemapFiemap' class is
507 returned. Otherwise, if 'SEEK_HOLE' is supported an instance of the
508 'FilemapSeek' class is returned. If none of these are supported, the
509 function generates an 'Error' type exception.
510 """
511
512 try:
513 return FilemapFiemap(image, log)
514 except ErrorNotSupp:
515 try:
516 return FilemapSeek(image, log)
517 except ErrorNotSupp:
518 return FilemapNobmap(image, log)
519
520def sparse_copy(src_fname, dst_fname, skip=0, seek=0,
521 length=0, api=None):
522 """
523 Efficiently copy sparse file to or into another file.
524
525 src_fname: path to source file
526 dst_fname: path to destination file
527 skip: skip N bytes at thestart of src
528 seek: seek N bytes from the start of dst
529 length: read N bytes from src and write them to dst
530 api: FilemapFiemap or FilemapSeek object
531 """
532 if not api:
533 api = filemap
534 fmap = api(src_fname)
535 try:
536 dst_file = open(dst_fname, 'r+b')
537 except IOError:
538 dst_file = open(dst_fname, 'wb')
539 if length:
540 dst_size = length + seek
541 else:
542 dst_size = os.path.getsize(src_fname) + seek - skip
543 dst_file.truncate(dst_size)
544
545 written = 0
546 for first, last in fmap.get_mapped_ranges(0, fmap.blocks_cnt):
547 start = first * fmap.block_size
548 end = (last + 1) * fmap.block_size
549
550 if skip >= end:
551 continue
552
553 if start < skip < end:
554 start = skip
555
556 fmap._f_image.seek(start, os.SEEK_SET)
557
558 written += start - skip - written
559 if length and written >= length:
560 dst_file.seek(seek + length, os.SEEK_SET)
561 dst_file.close()
562 return
563
564 dst_file.seek(seek + start - skip, os.SEEK_SET)
565
566 chunk_size = 1024 * 1024
567 to_read = end - start
568 read = 0
569
570 while read < to_read:
571 if read + chunk_size > to_read:
572 chunk_size = to_read - read
573 size = chunk_size
574 if length and written + size > length:
575 size = length - written
576 chunk = fmap._f_image.read(size)
577 dst_file.write(chunk)
578 read += size
579 written += size
580 if written == length:
581 dst_file.close()
582 return
583 dst_file.close()
diff --git a/meta-xilinx-core/scripts/lib/wic/help.py b/meta-xilinx-core/scripts/lib/wic/help.py
new file mode 100644
index 00000000..163535e4
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/help.py
@@ -0,0 +1,1148 @@
1# Copyright (c) 2013, Intel Corporation.
2#
3# SPDX-License-Identifier: GPL-2.0-only
4#
5# DESCRIPTION
6# This module implements some basic help invocation functions along
7# with the bulk of the help topic text for the OE Core Image Tools.
8#
9# AUTHORS
10# Tom Zanussi <tom.zanussi (at] linux.intel.com>
11#
12
13import subprocess
14import logging
15
16from wic.pluginbase import PluginMgr, PLUGIN_TYPES
17
18logger = logging.getLogger('wic')
19
20def subcommand_error(args):
21 logger.info("invalid subcommand %s", args[0])
22
23
24def display_help(subcommand, subcommands):
25 """
26 Display help for subcommand.
27 """
28 if subcommand not in subcommands:
29 return False
30
31 hlp = subcommands.get(subcommand, subcommand_error)[2]
32 if callable(hlp):
33 hlp = hlp()
34 pager = subprocess.Popen('less', stdin=subprocess.PIPE)
35 pager.communicate(hlp.encode('utf-8'))
36
37 return True
38
39
40def wic_help(args, usage_str, subcommands):
41 """
42 Subcommand help dispatcher.
43 """
44 if args.help_topic == None or not display_help(args.help_topic, subcommands):
45 print(usage_str)
46
47
48def get_wic_plugins_help():
49 """
50 Combine wic_plugins_help with the help for every known
51 source plugin.
52 """
53 result = wic_plugins_help
54 for plugin_type in PLUGIN_TYPES:
55 result += '\n\n%s PLUGINS\n\n' % plugin_type.upper()
56 for name, plugin in PluginMgr.get_plugins(plugin_type).items():
57 result += "\n %s plugin:\n" % name
58 if plugin.__doc__:
59 result += plugin.__doc__
60 else:
61 result += "\n %s is missing docstring\n" % plugin
62 return result
63
64
65def invoke_subcommand(args, parser, main_command_usage, subcommands):
66 """
67 Dispatch to subcommand handler borrowed from combo-layer.
68 Should use argparse, but has to work in 2.6.
69 """
70 if not args.command:
71 logger.error("No subcommand specified, exiting")
72 parser.print_help()
73 return 1
74 elif args.command == "help":
75 wic_help(args, main_command_usage, subcommands)
76 elif args.command not in subcommands:
77 logger.error("Unsupported subcommand %s, exiting\n", args.command)
78 parser.print_help()
79 return 1
80 else:
81 subcmd = subcommands.get(args.command, subcommand_error)
82 usage = subcmd[1]
83 subcmd[0](args, usage)
84
85
86##
87# wic help and usage strings
88##
89
90wic_usage = """
91
92 Create a customized OpenEmbedded image
93
94 usage: wic [--version] | [--help] | [COMMAND [ARGS]]
95
96 Current 'wic' commands are:
97 help Show help for command or one of the topics (see below)
98 create Create a new OpenEmbedded image
99 list List available canned images and source plugins
100
101 Help topics:
102 overview wic overview - General overview of wic
103 plugins wic plugins - Overview and API
104 kickstart wic kickstart - wic kickstart reference
105"""
106
107wic_help_usage = """
108
109 usage: wic help <subcommand>
110
111 This command displays detailed help for the specified subcommand.
112"""
113
114wic_create_usage = """
115
116 Create a new OpenEmbedded image
117
118 usage: wic create <wks file or image name> [-o <DIRNAME> | --outdir <DIRNAME>]
119 [-e | --image-name] [-s, --skip-build-check] [-D, --debug]
120 [-r, --rootfs-dir] [-b, --bootimg-dir]
121 [-k, --kernel-dir] [-n, --native-sysroot] [-f, --build-rootfs]
122 [-c, --compress-with] [-m, --bmap]
123
124 This command creates an OpenEmbedded image based on the 'OE kickstart
125 commands' found in the <wks file>.
126
127 The -o option can be used to place the image in a directory with a
128 different name and location.
129
130 See 'wic help create' for more detailed instructions.
131"""
132
133wic_create_help = """
134
135NAME
136 wic create - Create a new OpenEmbedded image
137
138SYNOPSIS
139 wic create <wks file or image name> [-o <DIRNAME> | --outdir <DIRNAME>]
140 [-e | --image-name] [-s, --skip-build-check] [-D, --debug]
141 [-r, --rootfs-dir] [-b, --bootimg-dir]
142 [-k, --kernel-dir] [-n, --native-sysroot] [-f, --build-rootfs]
143 [-c, --compress-with] [-m, --bmap] [--no-fstab-update]
144
145DESCRIPTION
146 This command creates an OpenEmbedded image based on the 'OE
147 kickstart commands' found in the <wks file>.
148
149 In order to do this, wic needs to know the locations of the
150 various build artifacts required to build the image.
151
152 Users can explicitly specify the build artifact locations using
153 the -r, -b, -k, and -n options. See below for details on where
154 the corresponding artifacts are typically found in a normal
155 OpenEmbedded build.
156
157 Alternatively, users can use the -e option to have 'wic' determine
158 those locations for a given image. If the -e option is used, the
159 user needs to have set the appropriate MACHINE variable in
160 local.conf, and have sourced the build environment.
161
162 The -e option is used to specify the name of the image to use the
163 artifacts from e.g. core-image-sato.
164
165 The -r option is used to specify the path to the /rootfs dir to
166 use as the .wks rootfs source.
167
168 The -b option is used to specify the path to the dir containing
169 the boot artifacts (e.g. /EFI or /syslinux dirs) to use as the
170 .wks bootimg source.
171
172 The -k option is used to specify the path to the dir containing
173 the kernel to use in the .wks bootimg.
174
175 The -n option is used to specify the path to the native sysroot
176 containing the tools to use to build the image.
177
178 The -f option is used to build rootfs by running "bitbake <image>"
179
180 The -s option is used to skip the build check. The build check is
181 a simple sanity check used to determine whether the user has
182 sourced the build environment so that the -e option can operate
183 correctly. If the user has specified the build artifact locations
184 explicitly, 'wic' assumes the user knows what he or she is doing
185 and skips the build check.
186
187 The -D option is used to display debug information detailing
188 exactly what happens behind the scenes when a create request is
189 fulfilled (or not, as the case may be). It enumerates and
190 displays the command sequence used, and should be included in any
191 bug report describing unexpected results.
192
193 When 'wic -e' is used, the locations for the build artifacts
194 values are determined by 'wic -e' from the output of the 'bitbake
195 -e' command given an image name e.g. 'core-image-minimal' and a
196 given machine set in local.conf. In that case, the image is
197 created as if the following 'bitbake -e' variables were used:
198
199 -r: IMAGE_ROOTFS
200 -k: STAGING_KERNEL_DIR
201 -n: STAGING_DIR_NATIVE
202 -b: empty (plugin-specific handlers must determine this)
203
204 If 'wic -e' is not used, the user needs to select the appropriate
205 value for -b (as well as -r, -k, and -n).
206
207 The -o option can be used to place the image in a directory with a
208 different name and location.
209
210 The -c option is used to specify compressor utility to compress
211 an image. gzip, bzip2 and xz compressors are supported.
212
213 The -m option is used to produce .bmap file for the image. This file
214 can be used to flash image using bmaptool utility.
215
216 The --no-fstab-update option is used to doesn't change fstab file. When
217 using this option the final fstab file will be same that in rootfs and
218 wic doesn't update file, e.g adding a new mount point. User can control
219 the fstab file content in base-files recipe.
220"""
221
222wic_list_usage = """
223
224 List available OpenEmbedded images and source plugins
225
226 usage: wic list images
227 wic list <image> help
228 wic list source-plugins
229
230 This command enumerates the set of available canned images as well as
231 help for those images. It also can be used to list of available source
232 plugins.
233
234 The first form enumerates all the available 'canned' images.
235
236 The second form lists the detailed help information for a specific
237 'canned' image.
238
239 The third form enumerates all the available --sources (source
240 plugins).
241
242 See 'wic help list' for more details.
243"""
244
245wic_list_help = """
246
247NAME
248 wic list - List available OpenEmbedded images and source plugins
249
250SYNOPSIS
251 wic list images
252 wic list <image> help
253 wic list source-plugins
254
255DESCRIPTION
256 This command enumerates the set of available canned images as well
257 as help for those images. It also can be used to list available
258 source plugins.
259
260 The first form enumerates all the available 'canned' images.
261 These are actually just the set of .wks files that have been moved
262 into the /scripts/lib/wic/canned-wks directory).
263
264 The second form lists the detailed help information for a specific
265 'canned' image.
266
267 The third form enumerates all the available --sources (source
268 plugins). The contents of a given partition are driven by code
269 defined in 'source plugins'. Users specify a specific plugin via
270 the --source parameter of the partition .wks command. Normally
271 this is the 'rootfs' plugin but can be any of the more specialized
272 sources listed by the 'list source-plugins' command. Users can
273 also add their own source plugins - see 'wic help plugins' for
274 details.
275"""
276
277wic_ls_usage = """
278
279 List content of a partitioned image
280
281 usage: wic ls <image>[:<partition>[<path>]] [--native-sysroot <path>]
282
283 This command outputs either list of image partitions or directory contents
284 of vfat and ext* partitions.
285
286 See 'wic help ls' for more detailed instructions.
287
288"""
289
290wic_ls_help = """
291
292NAME
293 wic ls - List contents of partitioned image or partition
294
295SYNOPSIS
296 wic ls <image>
297 wic ls <image>:<vfat or ext* partition>
298 wic ls <image>:<vfat or ext* partition><path>
299 wic ls <image>:<vfat or ext* partition><path> --native-sysroot <path>
300
301DESCRIPTION
302 This command lists either partitions of the image or directory contents
303 of vfat or ext* partitions.
304
305 The first form it lists partitions of the image.
306 For example:
307 $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic
308 Num Start End Size Fstype
309 1 1048576 24438783 23390208 fat16
310 2 25165824 50315263 25149440 ext4
311
312 Second and third form list directory content of the partition:
313 $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1
314 Volume in drive : is boot
315 Volume Serial Number is 2DF2-5F02
316 Directory for ::/
317
318 efi <DIR> 2017-05-11 10:54
319 startup nsh 26 2017-05-11 10:54
320 vmlinuz 6922288 2017-05-11 10:54
321 3 files 6 922 314 bytes
322 15 818 752 bytes free
323
324
325 $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/EFI/boot/
326 Volume in drive : is boot
327 Volume Serial Number is 2DF2-5F02
328 Directory for ::/EFI/boot
329
330 . <DIR> 2017-05-11 10:54
331 .. <DIR> 2017-05-11 10:54
332 grub cfg 679 2017-05-11 10:54
333 bootx64 efi 571392 2017-05-11 10:54
334 4 files 572 071 bytes
335 15 818 752 bytes free
336
337 The -n option is used to specify the path to the native sysroot
338 containing the tools(parted and mtools) to use.
339
340"""
341
342wic_cp_usage = """
343
344 Copy files and directories to/from the vfat or ext* partition
345
346 usage: wic cp <src> <dest> [--native-sysroot <path>]
347
348 source/destination image in format <image>:<partition>[<path>]
349
350 This command copies files or directories either
351 - from local to vfat or ext* partitions of partitioned image
352 - from vfat or ext* partitions of partitioned image to local
353
354 See 'wic help cp' for more detailed instructions.
355
356"""
357
358wic_cp_help = """
359
360NAME
361 wic cp - copy files and directories to/from the vfat or ext* partitions
362
363SYNOPSIS
364 wic cp <src> <dest>:<partition>
365 wic cp <src>:<partition> <dest>
366 wic cp <src> <dest-image>:<partition><path>
367 wic cp <src> <dest-image>:<partition><path> --native-sysroot <path>
368
369DESCRIPTION
370 This command copies files or directories either
371 - from local to vfat or ext* partitions of partitioned image
372 - from vfat or ext* partitions of partitioned image to local
373
374 The first form of it copies file or directory to the root directory of
375 the partition:
376 $ wic cp test.wks tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1
377 $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1
378 Volume in drive : is boot
379 Volume Serial Number is DB4C-FD4C
380 Directory for ::/
381
382 efi <DIR> 2017-05-24 18:15
383 loader <DIR> 2017-05-24 18:15
384 startup nsh 26 2017-05-24 18:15
385 vmlinuz 6926384 2017-05-24 18:15
386 test wks 628 2017-05-24 21:22
387 5 files 6 927 038 bytes
388 15 677 440 bytes free
389
390 The second form of the command copies file or directory to the specified directory
391 on the partition:
392 $ wic cp test tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/efi/
393 $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/efi/
394 Volume in drive : is boot
395 Volume Serial Number is DB4C-FD4C
396 Directory for ::/efi
397
398 . <DIR> 2017-05-24 18:15
399 .. <DIR> 2017-05-24 18:15
400 boot <DIR> 2017-05-24 18:15
401 test <DIR> 2017-05-24 21:27
402 4 files 0 bytes
403 15 675 392 bytes free
404
405 The third form of the command copies file or directory from the specified directory
406 on the partition to local:
407 $ wic cp tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/vmlinuz test
408
409 The -n option is used to specify the path to the native sysroot
410 containing the tools(parted and mtools) to use.
411"""
412
413wic_rm_usage = """
414
415 Remove files or directories from the vfat or ext* partitions
416
417 usage: wic rm <image>:<partition><path> [--native-sysroot <path>]
418
419 This command removes files or directories from the vfat or ext* partitions of
420 the partitioned image.
421
422 See 'wic help rm' for more detailed instructions.
423
424"""
425
426wic_rm_help = """
427
428NAME
429 wic rm - remove files or directories from the vfat or ext* partitions
430
431SYNOPSIS
432 wic rm <src> <image>:<partition><path>
433 wic rm <src> <image>:<partition><path> --native-sysroot <path>
434 wic rm -r <image>:<partition><path>
435
436DESCRIPTION
437 This command removes files or directories from the vfat or ext* partition of the
438 partitioned image:
439
440 $ wic ls ./tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1
441 Volume in drive : is boot
442 Volume Serial Number is 11D0-DE21
443 Directory for ::/
444
445 libcom32 c32 186500 2017-06-02 15:15
446 libutil c32 24148 2017-06-02 15:15
447 syslinux cfg 209 2017-06-02 15:15
448 vesamenu c32 27104 2017-06-02 15:15
449 vmlinuz 6926384 2017-06-02 15:15
450 5 files 7 164 345 bytes
451 16 582 656 bytes free
452
453 $ wic rm ./tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/libutil.c32
454
455 $ wic ls ./tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1
456 Volume in drive : is boot
457 Volume Serial Number is 11D0-DE21
458 Directory for ::/
459
460 libcom32 c32 186500 2017-06-02 15:15
461 syslinux cfg 209 2017-06-02 15:15
462 vesamenu c32 27104 2017-06-02 15:15
463 vmlinuz 6926384 2017-06-02 15:15
464 4 files 7 140 197 bytes
465 16 607 232 bytes free
466
467 The -n option is used to specify the path to the native sysroot
468 containing the tools(parted and mtools) to use.
469
470 The -r option is used to remove directories and their contents
471 recursively,this only applies to ext* partition.
472"""
473
474wic_write_usage = """
475
476 Write image to a device
477
478 usage: wic write <image> <target device> [--expand [rules]] [--native-sysroot <path>]
479
480 This command writes partitioned image to a target device (USB stick, SD card etc).
481
482 See 'wic help write' for more detailed instructions.
483
484"""
485
486wic_write_help = """
487
488NAME
489 wic write - write an image to a device
490
491SYNOPSIS
492 wic write <image> <target>
493 wic write <image> <target> --expand auto
494 wic write <image> <target> --expand 1:100M,2:300M
495 wic write <image> <target> --native-sysroot <path>
496
497DESCRIPTION
498 This command writes an image to a target device (USB stick, SD card etc)
499
500 $ wic write ./tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic /dev/sdb
501
502 The --expand option is used to resize image partitions.
503 --expand auto expands partitions to occupy all free space available on the target device.
504 It's also possible to specify expansion rules in a format
505 <partition>:<size>[,<partition>:<size>...] for one or more partitions.
506 Specifying size 0 will keep partition unmodified.
507 Note: Resizing boot partition can result in non-bootable image for non-EFI images. It is
508 recommended to use size 0 for boot partition to keep image bootable.
509
510 The --native-sysroot option is used to specify the path to the native sysroot
511 containing the tools(parted, resize2fs) to use.
512"""
513
514wic_plugins_help = """
515
516NAME
517 wic plugins - Overview and API
518
519DESCRIPTION
520 plugins allow wic functionality to be extended and specialized by
521 users. This section documents the plugin interface, which is
522 currently restricted to 'source' plugins.
523
524 'Source' plugins provide a mechanism to customize various aspects
525 of the image generation process in wic, mainly the contents of
526 partitions.
527
528 Source plugins provide a mechanism for mapping values specified in
529 .wks files using the --source keyword to a particular plugin
530 implementation that populates a corresponding partition.
531
532 A source plugin is created as a subclass of SourcePlugin (see
533 scripts/lib/wic/pluginbase.py) and the plugin file containing it
534 is added to scripts/lib/wic/plugins/source/ to make the plugin
535 implementation available to the wic implementation.
536
537 Source plugins can also be implemented and added by external
538 layers - any plugins found in a scripts/lib/wic/plugins/source/
539 or lib/wic/plugins/source/ directory in an external layer will
540 also be made available.
541
542 When the wic implementation needs to invoke a partition-specific
543 implementation, it looks for the plugin that has the same name as
544 the --source param given to that partition. For example, if the
545 partition is set up like this:
546
547 part /boot --source bootimg-pcbios ...
548
549 then the methods defined as class members of the plugin having the
550 matching bootimg-pcbios .name class member would be used.
551
552 To be more concrete, here's the plugin definition that would match
553 a '--source bootimg-pcbios' usage, along with an example method
554 that would be called by the wic implementation when it needed to
555 invoke an implementation-specific partition-preparation function:
556
557 class BootimgPcbiosPlugin(SourcePlugin):
558 name = 'bootimg-pcbios'
559
560 @classmethod
561 def do_prepare_partition(self, part, ...)
562
563 If the subclass itself doesn't implement a function, a 'default'
564 version in a superclass will be located and used, which is why all
565 plugins must be derived from SourcePlugin.
566
567 The SourcePlugin class defines the following methods, which is the
568 current set of methods that can be implemented/overridden by
569 --source plugins. Any methods not implemented by a SourcePlugin
570 subclass inherit the implementations present in the SourcePlugin
571 class (see the SourcePlugin source for details):
572
573 do_prepare_partition()
574 Called to do the actual content population for a
575 partition. In other words, it 'prepares' the final partition
576 image which will be incorporated into the disk image.
577
578 do_post_partition()
579 Called after the partition is created. It is useful to add post
580 operations e.g. signing the partition.
581
582 do_configure_partition()
583 Called before do_prepare_partition(), typically used to
584 create custom configuration files for a partition, for
585 example syslinux or grub config files.
586
587 do_install_disk()
588 Called after all partitions have been prepared and assembled
589 into a disk image. This provides a hook to allow
590 finalization of a disk image, for example to write an MBR to
591 it.
592
593 do_stage_partition()
594 Special content-staging hook called before
595 do_prepare_partition(), normally empty.
596
597 Typically, a partition will just use the passed-in
598 parameters, for example the unmodified value of bootimg_dir.
599 In some cases however, things may need to be more tailored.
600 As an example, certain files may additionally need to be
601 take from bootimg_dir + /boot. This hook allows those files
602 to be staged in a customized fashion. Note that
603 get_bitbake_var() allows you to access non-standard
604 variables that you might want to use for these types of
605 situations.
606
607 This scheme is extensible - adding more hooks is a simple matter
608 of adding more plugin methods to SourcePlugin and derived classes.
609 Please see the implementation for details.
610"""
611
612wic_overview_help = """
613
614NAME
615 wic overview - General overview of wic
616
617DESCRIPTION
618 The 'wic' command generates partitioned images from existing
619 OpenEmbedded build artifacts. Image generation is driven by
620 partitioning commands contained in an 'Openembedded kickstart'
621 (.wks) file (see 'wic help kickstart') specified either directly
622 on the command-line or as one of a selection of canned .wks files
623 (see 'wic list images'). When applied to a given set of build
624 artifacts, the result is an image or set of images that can be
625 directly written onto media and used on a particular system.
626
627 The 'wic' command and the infrastructure it's based on is by
628 definition incomplete - its purpose is to allow the generation of
629 customized images, and as such was designed to be completely
630 extensible via a plugin interface (see 'wic help plugins').
631
632 Background and Motivation
633
634 wic is meant to be a completely independent standalone utility
635 that initially provides easier-to-use and more flexible
636 replacements for a couple bits of existing functionality in
637 oe-core: directdisk.bbclass and mkefidisk.sh. The difference
638 between wic and those examples is that with wic the functionality
639 of those scripts is implemented by a general-purpose partitioning
640 'language' based on Red Hat kickstart syntax).
641
642 The initial motivation and design considerations that lead to the
643 current tool are described exhaustively in Yocto Bug #3847
644 (https://bugzilla.yoctoproject.org/show_bug.cgi?id=3847).
645
646 Implementation and Examples
647
648 wic can be used in two different modes, depending on how much
649 control the user needs in specifying the Openembedded build
650 artifacts that will be used in creating the image: 'raw' and
651 'cooked'.
652
653 If used in 'raw' mode, artifacts are explicitly specified via
654 command-line arguments (see example below).
655
656 The more easily usable 'cooked' mode uses the current MACHINE
657 setting and a specified image name to automatically locate the
658 artifacts used to create the image.
659
660 OE kickstart files (.wks) can of course be specified directly on
661 the command-line, but the user can also choose from a set of
662 'canned' .wks files available via the 'wic list images' command
663 (example below).
664
665 In any case, the prerequisite for generating any image is to have
666 the build artifacts already available. The below examples assume
667 the user has already build a 'core-image-minimal' for a specific
668 machine (future versions won't require this redundant step, but
669 for now that's typically how build artifacts get generated).
670
671 The other prerequisite is to source the build environment:
672
673 $ source oe-init-build-env
674
675 To start out with, we'll generate an image from one of the canned
676 .wks files. The following generates a list of availailable
677 images:
678
679 $ wic list images
680 mkefidisk Create an EFI disk image
681 directdisk Create a 'pcbios' direct disk image
682
683 You can get more information about any of the available images by
684 typing 'wic list xxx help', where 'xxx' is one of the image names:
685
686 $ wic list mkefidisk help
687
688 Creates a partitioned EFI disk image that the user can directly dd
689 to boot media.
690
691 At any time, you can get help on the 'wic' command or any
692 subcommand (currently 'list' and 'create'). For instance, to get
693 the description of 'wic create' command and its parameters:
694
695 $ wic create
696
697 Usage:
698
699 Create a new OpenEmbedded image
700
701 usage: wic create <wks file or image name> [-o <DIRNAME> | ...]
702 [-i <JSON PROPERTY FILE> | --infile <JSON PROPERTY_FILE>]
703 [-e | --image-name] [-s, --skip-build-check] [-D, --debug]
704 [-r, --rootfs-dir] [-b, --bootimg-dir] [-k, --kernel-dir]
705 [-n, --native-sysroot] [-f, --build-rootfs]
706
707 This command creates an OpenEmbedded image based on the 'OE
708 kickstart commands' found in the <wks file>.
709
710 The -o option can be used to place the image in a directory
711 with a different name and location.
712
713 See 'wic help create' for more detailed instructions.
714 ...
715
716 As mentioned in the command, you can get even more detailed
717 information by adding 'help' to the above:
718
719 $ wic help create
720
721 So, the easiest way to create an image is to use the -e option
722 with a canned .wks file. To use the -e option, you need to
723 specify the image used to generate the artifacts and you actually
724 need to have the MACHINE used to build them specified in your
725 local.conf (these requirements aren't necessary if you aren't
726 using the -e options.) Below, we generate a directdisk image,
727 pointing the process at the core-image-minimal artifacts for the
728 current MACHINE:
729
730 $ wic create directdisk -e core-image-minimal
731
732 Checking basic build environment...
733 Done.
734
735 Creating image(s)...
736
737 Info: The new image(s) can be found here:
738 /var/tmp/wic/build/directdisk-201309252350-sda.direct
739
740 The following build artifacts were used to create the image(s):
741
742 ROOTFS_DIR: ...
743 BOOTIMG_DIR: ...
744 KERNEL_DIR: ...
745 NATIVE_SYSROOT: ...
746
747 The image(s) were created using OE kickstart file:
748 .../scripts/lib/wic/canned-wks/directdisk.wks
749
750 The output shows the name and location of the image created, and
751 so that you know exactly what was used to generate the image, each
752 of the artifacts and the kickstart file used.
753
754 Similarly, you can create a 'mkefidisk' image in the same way
755 (notice that this example uses a different machine - because it's
756 using the -e option, you need to change the MACHINE in your
757 local.conf):
758
759 $ wic create mkefidisk -e core-image-minimal
760 Checking basic build environment...
761 Done.
762
763 Creating image(s)...
764
765 Info: The new image(s) can be found here:
766 /var/tmp/wic/build/mkefidisk-201309260027-sda.direct
767
768 ...
769
770 Here's an example that doesn't take the easy way out and manually
771 specifies each build artifact, along with a non-canned .wks file,
772 and also uses the -o option to have wic create the output
773 somewhere other than the default /var/tmp/wic:
774
775 $ wic create ./test.wks -o ./out --rootfs-dir
776 tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs
777 --bootimg-dir tmp/sysroots/qemux86-64/usr/share
778 --kernel-dir tmp/deploy/images/qemux86-64
779 --native-sysroot tmp/sysroots/x86_64-linux
780
781 Creating image(s)...
782
783 Info: The new image(s) can be found here:
784 out/build/test-201507211313-sda.direct
785
786 The following build artifacts were used to create the image(s):
787 ROOTFS_DIR: tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs
788 BOOTIMG_DIR: tmp/sysroots/qemux86-64/usr/share
789 KERNEL_DIR: tmp/deploy/images/qemux86-64
790 NATIVE_SYSROOT: tmp/sysroots/x86_64-linux
791
792 The image(s) were created using OE kickstart file:
793 ./test.wks
794
795 Here is a content of test.wks:
796
797 part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
798 part / --source rootfs --ondisk sda --fstype=ext3 --label platform --align 1024
799
800 bootloader --timeout=0 --append="rootwait rootfstype=ext3 video=vesafb vga=0x318 console=tty0"
801
802
803 Finally, here's an example of the actual partition language
804 commands used to generate the mkefidisk image i.e. these are the
805 contents of the mkefidisk.wks OE kickstart file:
806
807 # short-description: Create an EFI disk image
808 # long-description: Creates a partitioned EFI disk image that the user
809 # can directly dd to boot media.
810
811 part /boot --source bootimg-efi --ondisk sda --fstype=efi --active
812
813 part / --source rootfs --ondisk sda --fstype=ext3 --label platform
814
815 part swap --ondisk sda --size 44 --label swap1 --fstype=swap
816
817 bootloader --timeout=10 --append="rootwait console=ttyPCH0,115200"
818
819 You can get a complete listing and description of all the
820 kickstart commands available for use in .wks files from 'wic help
821 kickstart'.
822"""
823
824wic_kickstart_help = """
825
826NAME
827 wic kickstart - wic kickstart reference
828
829DESCRIPTION
830 This section provides the definitive reference to the wic
831 kickstart language. It also provides documentation on the list of
832 --source plugins available for use from the 'part' command (see
833 the 'Platform-specific Plugins' section below).
834
835 The current wic implementation supports only the basic kickstart
836 partitioning commands: partition (or part for short) and
837 bootloader.
838
839 The following is a listing of the commands, their syntax, and
840 meanings. The commands are based on the Fedora kickstart
841 documentation but with modifications to reflect wic capabilities.
842
843 https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#part-or-partition
844 https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#bootloader
845
846 Commands
847
848 * 'part' or 'partition'
849
850 This command creates a partition on the system and uses the
851 following syntax:
852
853 part [<mountpoint>]
854
855 The <mountpoint> is where the partition will be mounted and
856 must take of one of the following forms:
857
858 /<path>: For example: /, /usr, or /home
859
860 swap: The partition will be used as swap space.
861
862 If a <mountpoint> is not specified the partition will be created
863 but will not be mounted.
864
865 Partitions with a <mountpoint> specified will be automatically mounted.
866 This is achieved by wic adding entries to the fstab during image
867 generation. In order for a valid fstab to be generated one of the
868 --ondrive, --ondisk, --use-uuid or --use-label partition options must
869 be used for each partition that specifies a mountpoint. Note that with
870 --use-{uuid,label} and non-root <mountpoint>, including swap, the mount
871 program must understand the PARTUUID or LABEL syntax. This currently
872 excludes the busybox versions of these applications.
873
874
875 The following are supported 'part' options:
876
877 --size: The minimum partition size. Specify an integer value
878 such as 500. Multipliers k, M ang G can be used. If
879 not specified, the size is in MB.
880 You do not need this option if you use --source.
881
882 --fixed-size: Exact partition size. Value format is the same
883 as for --size option. This option cannot be
884 specified along with --size. If partition data
885 is larger than --fixed-size and error will be
886 raised when assembling disk image.
887
888 --source: This option is a wic-specific option that names the
889 source of the data that will populate the
890 partition. The most common value for this option
891 is 'rootfs', but can be any value which maps to a
892 valid 'source plugin' (see 'wic help plugins').
893
894 If '--source rootfs' is used, it tells the wic
895 command to create a partition as large as needed
896 and to fill it with the contents of the root
897 filesystem pointed to by the '-r' wic command-line
898 option (or the equivalent rootfs derived from the
899 '-e' command-line option). The filesystem type
900 that will be used to create the partition is driven
901 by the value of the --fstype option specified for
902 the partition (see --fstype below).
903
904 If --source <plugin-name>' is used, it tells the
905 wic command to create a partition as large as
906 needed and to fill with the contents of the
907 partition that will be generated by the specified
908 plugin name using the data pointed to by the '-r'
909 wic command-line option (or the equivalent rootfs
910 derived from the '-e' command-line option).
911 Exactly what those contents and filesystem type end
912 up being are dependent on the given plugin
913 implementation.
914
915 If --source option is not used, the wic command
916 will create empty partition. --size parameter has
917 to be used to specify size of empty partition.
918
919 --ondisk or --ondrive: Forces the partition to be created on
920 a particular disk.
921
922 --fstype: Sets the file system type for the partition. These
923 apply to partitions created using '--source rootfs' (see
924 --source above). Valid values are:
925
926 vfat
927 msdos
928 ext2
929 ext3
930 ext4
931 btrfs
932 squashfs
933 erofs
934 swap
935
936 --fsoptions: Specifies a free-form string of options to be
937 used when mounting the filesystem. This string
938 will be copied into the /etc/fstab file of the
939 installed system and should be enclosed in
940 quotes. If not specified, the default string is
941 "defaults".
942
943 --fspassno: Specifies the order in which filesystem checks are done
944 at boot time by fsck. See fs_passno parameter of
945 fstab(5). This parameter will be copied into the
946 /etc/fstab file of the installed system. If not
947 specified the default value of "0" will be used.
948
949 --label label: Specifies the label to give to the filesystem
950 to be made on the partition. If the given
951 label is already in use by another filesystem,
952 a new label is created for the partition.
953
954 --use-label: This option is specific to wic. It makes wic to use the
955 label in /etc/fstab to specify a partition. If the
956 --use-label and --use-uuid are used at the same time,
957 we prefer the uuid because it is less likely to cause
958 name confliction. We don't support using this parameter
959 on the root partition since it requires an initramfs to
960 parse this value and we do not currently support that.
961
962 --active: Marks the partition as active.
963
964 --align (in KBytes): This option is specific to wic and says
965 to start a partition on an x KBytes
966 boundary.
967
968 --no-table: This option is specific to wic. Space will be
969 reserved for the partition and it will be
970 populated but it will not be added to the
971 partition table. It may be useful for
972 bootloaders.
973
974 --exclude-path: This option is specific to wic. It excludes the given
975 relative path from the resulting image. If the path
976 ends with a slash, only the content of the directory
977 is omitted, not the directory itself. This option only
978 has an effect with the rootfs source plugin.
979
980 --include-path: This option is specific to wic. It adds the contents
981 of the given path or a rootfs to the resulting image.
982 The option contains two fields, the origin and the
983 destination. When the origin is a rootfs, it follows
984 the same logic as the rootfs-dir argument and the
985 permissions and owners are kept. When the origin is a
986 path, it is relative to the directory in which wic is
987 running not the rootfs itself so use of an absolute
988 path is recommended, and the owner and group is set to
989 root:root. If no destination is given it is
990 automatically set to the root of the rootfs. This
991 option only has an effect with the rootfs source
992 plugin.
993
994 --change-directory: This option is specific to wic. It changes to the
995 given directory before copying the files. This
996 option is useful when we want to split a rootfs in
997 multiple partitions and we want to keep the right
998 permissions and usernames in all the partitions.
999
1000 --no-fstab-update: This option is specific to wic. It does not update the
1001 '/etc/fstab' stock file for the given partition.
1002
1003 --extra-space: This option is specific to wic. It adds extra
1004 space after the space filled by the content
1005 of the partition. The final size can go
1006 beyond the size specified by --size.
1007 By default, 10MB. This option cannot be used
1008 with --fixed-size option.
1009
1010 --overhead-factor: This option is specific to wic. The
1011 size of the partition is multiplied by
1012 this factor. It has to be greater than or
1013 equal to 1. The default value is 1.3.
1014 This option cannot be used with --fixed-size
1015 option.
1016
1017 --part-name: This option is specific to wic. It specifies name for GPT partitions.
1018
1019 --part-type: This option is specific to wic. It specifies partition
1020 type GUID for GPT partitions.
1021 List of partition type GUIDS can be found here:
1022 http://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_type_GUIDs
1023
1024 --use-uuid: This option is specific to wic. It makes wic to generate
1025 random globally unique identifier (GUID) for the partition
1026 and use it in bootloader configuration to specify root partition.
1027
1028 --uuid: This option is specific to wic. It specifies partition UUID.
1029 It's useful if preconfigured partition UUID is added to kernel command line
1030 in bootloader configuration before running wic. In this case .wks file can
1031 be generated or modified to set preconfigured parition UUID using this option.
1032
1033 --fsuuid: This option is specific to wic. It specifies filesystem UUID.
1034 It's useful if preconfigured filesystem UUID is added to kernel command line
1035 in bootloader configuration before running wic. In this case .wks file can
1036 be generated or modified to set preconfigured filesystem UUID using this option.
1037
1038 --system-id: This option is specific to wic. It specifies partition system id. It's useful
1039 for the harware that requires non-default partition system ids. The parameter
1040 in one byte long hex number either with 0x prefix or without it.
1041
1042 --mkfs-extraopts: This option specifies extra options to pass to mkfs utility.
1043 NOTE, that wic uses default options for some filesystems, for example
1044 '-S 512' for mkfs.fat or '-F -i 8192' for mkfs.ext. Those options will
1045 not take effect when --mkfs-extraopts is used. This should be taken into
1046 account when using --mkfs-extraopts.
1047
1048 * bootloader
1049
1050 This command allows the user to specify various bootloader
1051 options. The following are supported 'bootloader' options:
1052
1053 --timeout: Specifies the number of seconds before the
1054 bootloader times out and boots the default option.
1055
1056 --append: Specifies kernel parameters. These will be added to
1057 bootloader command-line - for example, the syslinux
1058 APPEND or grub kernel command line.
1059
1060 --configfile: Specifies a user defined configuration file for
1061 the bootloader. This file must be located in the
1062 canned-wks folder or could be the full path to the
1063 file. Using this option will override any other
1064 bootloader option.
1065
1066 Note that bootloader functionality and boot partitions are
1067 implemented by the various --source plugins that implement
1068 bootloader functionality; the bootloader command essentially
1069 provides a means of modifying bootloader configuration.
1070
1071 * include
1072
1073 This command allows the user to include the content of .wks file
1074 into original .wks file.
1075
1076 Command uses the following syntax:
1077
1078 include <file>
1079
1080 The <file> is either path to the file or its name. If name is
1081 specified wic will try to find file in the directories with canned
1082 .wks files.
1083
1084"""
1085
1086wic_help_help = """
1087NAME
1088 wic help - display a help topic
1089
1090DESCRIPTION
1091 Specify a help topic to display it. Topics are shown above.
1092"""
1093
1094
1095wic_help = """
1096Creates a customized OpenEmbedded image.
1097
1098Usage: wic [--version]
1099 wic help [COMMAND or TOPIC]
1100 wic COMMAND [ARGS]
1101
1102 usage 1: Returns the current version of Wic
1103 usage 2: Returns detailed help for a COMMAND or TOPIC
1104 usage 3: Executes COMMAND
1105
1106
1107COMMAND:
1108
1109 list - List available canned images and source plugins
1110 ls - List contents of partitioned image or partition
1111 rm - Remove files or directories from the vfat or ext* partitions
1112 help - Show help for a wic COMMAND or TOPIC
1113 write - Write an image to a device
1114 cp - Copy files and directories to the vfat or ext* partitions
1115 create - Create a new OpenEmbedded image
1116
1117
1118TOPIC:
1119 overview - Presents an overall overview of Wic
1120 plugins - Presents an overview and API for Wic plugins
1121 kickstart - Presents a Wic kickstart file reference
1122
1123
1124Examples:
1125
1126 $ wic --version
1127
1128 Returns the current version of Wic
1129
1130
1131 $ wic help cp
1132
1133 Returns the SYNOPSIS and DESCRIPTION for the Wic "cp" command.
1134
1135
1136 $ wic list images
1137
1138 Returns the list of canned images (i.e. *.wks files located in
1139 the /scripts/lib/wic/canned-wks directory.
1140
1141
1142 $ wic create mkefidisk -e core-image-minimal
1143
1144 Creates an EFI disk image from artifacts used in a previous
1145 core-image-minimal build in standard BitBake locations
1146 (e.g. Cooked Mode).
1147
1148"""
diff --git a/meta-xilinx-core/scripts/lib/wic/ksparser.py b/meta-xilinx-core/scripts/lib/wic/ksparser.py
new file mode 100644
index 00000000..7ef3dc83
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/ksparser.py
@@ -0,0 +1,298 @@
1#!/usr/bin/env python3
2#
3# Copyright (c) 2016 Intel, Inc.
4#
5# SPDX-License-Identifier: GPL-2.0-only
6#
7# DESCRIPTION
8# This module provides parser for kickstart format
9#
10# AUTHORS
11# Ed Bartosh <ed.bartosh> (at] linux.intel.com>
12
13"""Kickstart parser module."""
14
15import os
16import shlex
17import logging
18import re
19
20from argparse import ArgumentParser, ArgumentError, ArgumentTypeError
21
22from wic.engine import find_canned
23from wic.partition import Partition
24from wic.misc import get_bitbake_var
25
26logger = logging.getLogger('wic')
27
28__expand_var_regexp__ = re.compile(r"\${[^{}@\n\t :]+}")
29
30def expand_line(line):
31 while True:
32 m = __expand_var_regexp__.search(line)
33 if not m:
34 return line
35 key = m.group()[2:-1]
36 val = get_bitbake_var(key)
37 if val is None:
38 logger.warning("cannot expand variable %s" % key)
39 return line
40 line = line[:m.start()] + val + line[m.end():]
41
42class KickStartError(Exception):
43 """Custom exception."""
44 pass
45
46class KickStartParser(ArgumentParser):
47 """
48 This class overwrites error method to throw exception
49 instead of producing usage message(default argparse behavior).
50 """
51 def error(self, message):
52 raise ArgumentError(None, message)
53
54def sizetype(default, size_in_bytes=False):
55 def f(arg):
56 """
57 Custom type for ArgumentParser
58 Converts size string in <num>[S|s|K|k|M|G] format into the integer value
59 """
60 try:
61 suffix = default
62 size = int(arg)
63 except ValueError:
64 try:
65 suffix = arg[-1:]
66 size = int(arg[:-1])
67 except ValueError:
68 raise ArgumentTypeError("Invalid size: %r" % arg)
69
70
71 if size_in_bytes:
72 if suffix == 's' or suffix == 'S':
73 return size * 512
74 mult = 1024
75 else:
76 mult = 1
77
78 if suffix == "k" or suffix == "K":
79 return size * mult
80 if suffix == "M":
81 return size * mult * 1024
82 if suffix == "G":
83 return size * mult * 1024 * 1024
84
85 raise ArgumentTypeError("Invalid size: %r" % arg)
86 return f
87
88def overheadtype(arg):
89 """
90 Custom type for ArgumentParser
91 Converts overhead string to float and checks if it's bigger than 1.0
92 """
93 try:
94 result = float(arg)
95 except ValueError:
96 raise ArgumentTypeError("Invalid value: %r" % arg)
97
98 if result < 1.0:
99 raise ArgumentTypeError("Overhead factor should be > 1.0" % arg)
100
101 return result
102
103def cannedpathtype(arg):
104 """
105 Custom type for ArgumentParser
106 Tries to find file in the list of canned wks paths
107 """
108 scripts_path = os.path.abspath(os.path.dirname(__file__) + '../../..')
109 result = find_canned(scripts_path, arg)
110 if not result:
111 raise ArgumentTypeError("file not found: %s" % arg)
112 return result
113
114def systemidtype(arg):
115 """
116 Custom type for ArgumentParser
117 Checks if the argument sutisfies system id requirements,
118 i.e. if it's one byte long integer > 0
119 """
120 error = "Invalid system type: %s. must be hex "\
121 "between 0x1 and 0xFF" % arg
122 try:
123 result = int(arg, 16)
124 except ValueError:
125 raise ArgumentTypeError(error)
126
127 if result <= 0 or result > 0xff:
128 raise ArgumentTypeError(error)
129
130 return arg
131
132class KickStart():
133 """Kickstart parser implementation."""
134
135 DEFAULT_EXTRA_SPACE = 10*1024
136 DEFAULT_OVERHEAD_FACTOR = 1.3
137
138 def __init__(self, confpath):
139
140 self.partitions = []
141 self.bootloader = None
142 self.lineno = 0
143 self.partnum = 0
144
145 parser = KickStartParser()
146 subparsers = parser.add_subparsers()
147
148 part = subparsers.add_parser('part')
149 part.add_argument('mountpoint', nargs='?')
150 part.add_argument('--active', action='store_true')
151 part.add_argument('--align', type=int)
152 part.add_argument('--offset', type=sizetype("K", True))
153 part.add_argument('--exclude-path', nargs='+')
154 part.add_argument('--include-path', nargs='+', action='append')
155 part.add_argument('--change-directory')
156 part.add_argument("--extra-space", type=sizetype("M"))
157 part.add_argument('--fsoptions', dest='fsopts')
158 part.add_argument('--fspassno', dest='fspassno')
159 part.add_argument('--fstype', default='vfat',
160 choices=('ext2', 'ext3', 'ext4', 'btrfs',
161 'squashfs', 'vfat', 'msdos', 'erofs',
162 'swap', 'none'))
163 part.add_argument('--mkfs-extraopts', default='')
164 part.add_argument('--label')
165 part.add_argument('--use-label', action='store_true')
166 part.add_argument('--no-table', action='store_true')
167 part.add_argument('--ondisk', '--ondrive', dest='disk', default='sda')
168 part.add_argument("--overhead-factor", type=overheadtype)
169 part.add_argument('--part-name')
170 part.add_argument('--part-type')
171 part.add_argument('--rootfs-dir')
172 part.add_argument('--type', default='primary',
173 choices = ('primary', 'logical'))
174 part.add_argument('--hidden', action='store_true')
175
176 # --size and --fixed-size cannot be specified together; options
177 # ----extra-space and --overhead-factor should also raise a parser
178 # --error, but since nesting mutually exclusive groups does not work,
179 # ----extra-space/--overhead-factor are handled later
180 sizeexcl = part.add_mutually_exclusive_group()
181 sizeexcl.add_argument('--size', type=sizetype("M"), default=0)
182 sizeexcl.add_argument('--fixed-size', type=sizetype("M"), default=0)
183
184 part.add_argument('--source')
185 part.add_argument('--sourceparams')
186 part.add_argument('--system-id', type=systemidtype)
187 part.add_argument('--use-uuid', action='store_true')
188 part.add_argument('--uuid')
189 part.add_argument('--fsuuid')
190 part.add_argument('--no-fstab-update', action='store_true')
191 part.add_argument('--mbr', action='store_true')
192
193 bootloader = subparsers.add_parser('bootloader')
194 bootloader.add_argument('--append')
195 bootloader.add_argument('--configfile')
196 bootloader.add_argument('--ptable', choices=('msdos', 'gpt', 'gpt-hybrid'),
197 default='msdos')
198 bootloader.add_argument('--timeout', type=int)
199 bootloader.add_argument('--source')
200
201 include = subparsers.add_parser('include')
202 include.add_argument('path', type=cannedpathtype)
203
204 self._parse(parser, confpath)
205 if not self.bootloader:
206 logger.warning('bootloader config not specified, using defaults\n')
207 self.bootloader = bootloader.parse_args([])
208
209 def _parse(self, parser, confpath):
210 """
211 Parse file in .wks format using provided parser.
212 """
213 with open(confpath) as conf:
214 lineno = 0
215 for line in conf:
216 line = line.strip()
217 lineno += 1
218 if line and line[0] != '#':
219 line = expand_line(line)
220 try:
221 line_args = shlex.split(line)
222 parsed = parser.parse_args(line_args)
223 except ArgumentError as err:
224 raise KickStartError('%s:%d: %s' % \
225 (confpath, lineno, err))
226 if line.startswith('part'):
227 # SquashFS does not support filesystem UUID
228 if parsed.fstype == 'squashfs':
229 if parsed.fsuuid:
230 err = "%s:%d: SquashFS does not support UUID" \
231 % (confpath, lineno)
232 raise KickStartError(err)
233 if parsed.label:
234 err = "%s:%d: SquashFS does not support LABEL" \
235 % (confpath, lineno)
236 raise KickStartError(err)
237 # erofs does not support filesystem labels
238 if parsed.fstype == 'erofs' and parsed.label:
239 err = "%s:%d: erofs does not support LABEL" % (confpath, lineno)
240 raise KickStartError(err)
241 if parsed.fstype == 'msdos' or parsed.fstype == 'vfat':
242 if parsed.fsuuid:
243 if parsed.fsuuid.upper().startswith('0X'):
244 if len(parsed.fsuuid) > 10:
245 err = "%s:%d: fsuuid %s given in wks kickstart file " \
246 "exceeds the length limit for %s filesystem. " \
247 "It should be in the form of a 32 bit hexadecimal" \
248 "number (for example, 0xABCD1234)." \
249 % (confpath, lineno, parsed.fsuuid, parsed.fstype)
250 raise KickStartError(err)
251 elif len(parsed.fsuuid) > 8:
252 err = "%s:%d: fsuuid %s given in wks kickstart file " \
253 "exceeds the length limit for %s filesystem. " \
254 "It should be in the form of a 32 bit hexadecimal" \
255 "number (for example, 0xABCD1234)." \
256 % (confpath, lineno, parsed.fsuuid, parsed.fstype)
257 raise KickStartError(err)
258 if parsed.use_label and not parsed.label:
259 err = "%s:%d: Must set the label with --label" \
260 % (confpath, lineno)
261 raise KickStartError(err)
262 # using ArgumentParser one cannot easily tell if option
263 # was passed as argument, if said option has a default
264 # value; --overhead-factor/--extra-space cannot be used
265 # with --fixed-size, so at least detect when these were
266 # passed with non-0 values ...
267 if parsed.fixed_size:
268 if parsed.overhead_factor or parsed.extra_space:
269 err = "%s:%d: arguments --overhead-factor and --extra-space not "\
270 "allowed with argument --fixed-size" \
271 % (confpath, lineno)
272 raise KickStartError(err)
273 else:
274 # ... and provide defaults if not using
275 # --fixed-size iff given option was not used
276 # (again, one cannot tell if option was passed but
277 # with value equal to 0)
278 if '--overhead-factor' not in line_args:
279 parsed.overhead_factor = self.DEFAULT_OVERHEAD_FACTOR
280 if '--extra-space' not in line_args:
281 parsed.extra_space = self.DEFAULT_EXTRA_SPACE
282
283 self.partnum += 1
284 self.partitions.append(Partition(parsed, self.partnum))
285 elif line.startswith('include'):
286 self._parse(parser, parsed.path)
287 elif line.startswith('bootloader'):
288 if not self.bootloader:
289 self.bootloader = parsed
290 # Concatenate the strings set in APPEND
291 append_var = get_bitbake_var("APPEND")
292 if append_var:
293 self.bootloader.append = ' '.join(filter(None, \
294 (self.bootloader.append, append_var)))
295 else:
296 err = "%s:%d: more than one bootloader specified" \
297 % (confpath, lineno)
298 raise KickStartError(err)
diff --git a/meta-xilinx-core/scripts/lib/wic/misc.py b/meta-xilinx-core/scripts/lib/wic/misc.py
new file mode 100644
index 00000000..1a7c140f
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/misc.py
@@ -0,0 +1,266 @@
1#
2# Copyright (c) 2013, Intel Corporation.
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6# DESCRIPTION
7# This module provides a place to collect various wic-related utils
8# for the OpenEmbedded Image Tools.
9#
10# AUTHORS
11# Tom Zanussi <tom.zanussi (at] linux.intel.com>
12#
13"""Miscellaneous functions."""
14
15import logging
16import os
17import re
18import subprocess
19import shutil
20
21from collections import defaultdict
22
23from wic import WicError
24
25logger = logging.getLogger('wic')
26
27# executable -> recipe pairs for exec_native_cmd
28NATIVE_RECIPES = {"bmaptool": "bmaptool",
29 "dumpe2fs": "e2fsprogs",
30 "grub-mkimage": "grub-efi",
31 "isohybrid": "syslinux",
32 "mcopy": "mtools",
33 "mdel" : "mtools",
34 "mdeltree" : "mtools",
35 "mdir" : "mtools",
36 "mkdosfs": "dosfstools",
37 "mkisofs": "cdrtools",
38 "mkfs.btrfs": "btrfs-tools",
39 "mkfs.erofs": "erofs-utils",
40 "mkfs.ext2": "e2fsprogs",
41 "mkfs.ext3": "e2fsprogs",
42 "mkfs.ext4": "e2fsprogs",
43 "mkfs.vfat": "dosfstools",
44 "mksquashfs": "squashfs-tools",
45 "mkswap": "util-linux",
46 "mmd": "mtools",
47 "parted": "parted",
48 "sfdisk": "util-linux",
49 "sgdisk": "gptfdisk",
50 "syslinux": "syslinux",
51 "tar": "tar"
52 }
53
54def runtool(cmdln_or_args):
55 """ wrapper for most of the subprocess calls
56 input:
57 cmdln_or_args: can be both args and cmdln str (shell=True)
58 return:
59 rc, output
60 """
61 if isinstance(cmdln_or_args, list):
62 cmd = cmdln_or_args[0]
63 shell = False
64 else:
65 import shlex
66 cmd = shlex.split(cmdln_or_args)[0]
67 shell = True
68
69 sout = subprocess.PIPE
70 serr = subprocess.STDOUT
71
72 try:
73 process = subprocess.Popen(cmdln_or_args, stdout=sout,
74 stderr=serr, shell=shell)
75 sout, serr = process.communicate()
76 # combine stdout and stderr, filter None out and decode
77 out = ''.join([out.decode('utf-8') for out in [sout, serr] if out])
78 except OSError as err:
79 if err.errno == 2:
80 # [Errno 2] No such file or directory
81 raise WicError('Cannot run command: %s, lost dependency?' % cmd)
82 else:
83 raise # relay
84
85 return process.returncode, out
86
87def _exec_cmd(cmd_and_args, as_shell=False):
88 """
89 Execute command, catching stderr, stdout
90
91 Need to execute as_shell if the command uses wildcards
92 """
93 logger.debug("_exec_cmd: %s", cmd_and_args)
94 args = cmd_and_args.split()
95 logger.debug(args)
96
97 if as_shell:
98 ret, out = runtool(cmd_and_args)
99 else:
100 ret, out = runtool(args)
101 out = out.strip()
102 if ret != 0:
103 raise WicError("_exec_cmd: %s returned '%s' instead of 0\noutput: %s" % \
104 (cmd_and_args, ret, out))
105
106 logger.debug("_exec_cmd: output for %s (rc = %d): %s",
107 cmd_and_args, ret, out)
108
109 return ret, out
110
111
112def exec_cmd(cmd_and_args, as_shell=False):
113 """
114 Execute command, return output
115 """
116 return _exec_cmd(cmd_and_args, as_shell)[1]
117
118def find_executable(cmd, paths):
119 recipe = cmd
120 if recipe in NATIVE_RECIPES:
121 recipe = NATIVE_RECIPES[recipe]
122 provided = get_bitbake_var("ASSUME_PROVIDED")
123 if provided and "%s-native" % recipe in provided:
124 return True
125
126 return shutil.which(cmd, path=paths)
127
128def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""):
129 """
130 Execute native command, catching stderr, stdout
131
132 Need to execute as_shell if the command uses wildcards
133
134 Always need to execute native commands as_shell
135 """
136 # The reason -1 is used is because there may be "export" commands.
137 args = cmd_and_args.split(';')[-1].split()
138 logger.debug(args)
139
140 if pseudo:
141 cmd_and_args = pseudo + cmd_and_args
142
143 hosttools_dir = get_bitbake_var("HOSTTOOLS_DIR")
144 target_sys = get_bitbake_var("TARGET_SYS")
145
146 native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/usr/bin/%s:%s/bin:%s" % \
147 (native_sysroot, native_sysroot,
148 native_sysroot, native_sysroot, target_sys,
149 native_sysroot, hosttools_dir)
150
151 native_cmd_and_args = "export PATH=%s:$PATH;%s" % \
152 (native_paths, cmd_and_args)
153 logger.debug("exec_native_cmd: %s", native_cmd_and_args)
154
155 # If the command isn't in the native sysroot say we failed.
156 if find_executable(args[0], native_paths):
157 ret, out = _exec_cmd(native_cmd_and_args, True)
158 else:
159 ret = 127
160 out = "can't find native executable %s in %s" % (args[0], native_paths)
161
162 prog = args[0]
163 # shell command-not-found
164 if ret == 127 \
165 or (pseudo and ret == 1 and out == "Can't find '%s' in $PATH." % prog):
166 msg = "A native program %s required to build the image "\
167 "was not found (see details above).\n\n" % prog
168 recipe = NATIVE_RECIPES.get(prog)
169 if recipe:
170 msg += "Please make sure wic-tools have %s-native in its DEPENDS, "\
171 "build it with 'bitbake wic-tools' and try again.\n" % recipe
172 else:
173 msg += "Wic failed to find a recipe to build native %s. Please "\
174 "file a bug against wic.\n" % prog
175 raise WicError(msg)
176
177 return ret, out
178
179BOOTDD_EXTRA_SPACE = 16384
180
181class BitbakeVars(defaultdict):
182 """
183 Container for Bitbake variables.
184 """
185 def __init__(self):
186 defaultdict.__init__(self, dict)
187
188 # default_image and vars_dir attributes should be set from outside
189 self.default_image = None
190 self.vars_dir = None
191
192 def _parse_line(self, line, image, matcher=re.compile(r"^([a-zA-Z0-9\-_+./~]+)=(.*)")):
193 """
194 Parse one line from bitbake -e output or from .env file.
195 Put result key-value pair into the storage.
196 """
197 if "=" not in line:
198 return
199 match = matcher.match(line)
200 if not match:
201 return
202 key, val = match.groups()
203 self[image][key] = val.strip('"')
204
205 def get_var(self, var, image=None, cache=True):
206 """
207 Get bitbake variable from 'bitbake -e' output or from .env file.
208 This is a lazy method, i.e. it runs bitbake or parses file only when
209 only when variable is requested. It also caches results.
210 """
211 if not image:
212 image = self.default_image
213
214 if image not in self:
215 if image and self.vars_dir:
216 fname = os.path.join(self.vars_dir, image + '.env')
217 if os.path.isfile(fname):
218 # parse .env file
219 with open(fname) as varsfile:
220 for line in varsfile:
221 self._parse_line(line, image)
222 else:
223 print("Couldn't get bitbake variable from %s." % fname)
224 print("File %s doesn't exist." % fname)
225 return
226 else:
227 # Get bitbake -e output
228 cmd = "bitbake -e"
229 if image:
230 cmd += " %s" % image
231
232 log_level = logger.getEffectiveLevel()
233 logger.setLevel(logging.INFO)
234 ret, lines = _exec_cmd(cmd)
235 logger.setLevel(log_level)
236
237 if ret:
238 logger.error("Couldn't get '%s' output.", cmd)
239 logger.error("Bitbake failed with error:\n%s\n", lines)
240 return
241
242 # Parse bitbake -e output
243 for line in lines.split('\n'):
244 self._parse_line(line, image)
245
246 # Make first image a default set of variables
247 if cache:
248 images = [key for key in self if key]
249 if len(images) == 1:
250 self[None] = self[image]
251
252 result = self[image].get(var)
253 if not cache:
254 self.pop(image, None)
255
256 return result
257
258# Create BB_VARS singleton
259BB_VARS = BitbakeVars()
260
261def get_bitbake_var(var, image=None, cache=True):
262 """
263 Provide old get_bitbake_var API by wrapping
264 get_var method of BB_VARS singleton.
265 """
266 return BB_VARS.get_var(var, image, cache)
diff --git a/meta-xilinx-core/scripts/lib/wic/partition.py b/meta-xilinx-core/scripts/lib/wic/partition.py
new file mode 100644
index 00000000..bf2c34d5
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/partition.py
@@ -0,0 +1,551 @@
1#
2# Copyright (c) 2013-2016 Intel Corporation.
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6# DESCRIPTION
7# This module provides the OpenEmbedded partition object definitions.
8#
9# AUTHORS
10# Tom Zanussi <tom.zanussi (at] linux.intel.com>
11# Ed Bartosh <ed.bartosh> (at] linux.intel.com>
12
13import logging
14import os
15import uuid
16
17from wic import WicError
18from wic.misc import exec_cmd, exec_native_cmd, get_bitbake_var
19from wic.pluginbase import PluginMgr
20
21logger = logging.getLogger('wic')
22
23class Partition():
24
25 def __init__(self, args, lineno):
26 self.args = args
27 self.active = args.active
28 self.align = args.align
29 self.disk = args.disk
30 self.device = None
31 self.extra_space = args.extra_space
32 self.exclude_path = args.exclude_path
33 self.include_path = args.include_path
34 self.change_directory = args.change_directory
35 self.fsopts = args.fsopts
36 self.fspassno = args.fspassno
37 self.fstype = args.fstype
38 self.label = args.label
39 self.use_label = args.use_label
40 self.mkfs_extraopts = args.mkfs_extraopts
41 self.mountpoint = args.mountpoint
42 self.no_table = args.no_table
43 self.num = None
44 self.offset = args.offset
45 self.overhead_factor = args.overhead_factor
46 self.part_name = args.part_name
47 self.part_type = args.part_type
48 self.rootfs_dir = args.rootfs_dir
49 self.size = args.size
50 self.fixed_size = args.fixed_size
51 self.source = args.source
52 self.sourceparams = args.sourceparams
53 self.system_id = args.system_id
54 self.use_uuid = args.use_uuid
55 self.uuid = args.uuid
56 self.fsuuid = args.fsuuid
57 self.type = args.type
58 self.no_fstab_update = args.no_fstab_update
59 self.updated_fstab_path = None
60 self.has_fstab = False
61 self.update_fstab_in_rootfs = False
62 self.hidden = args.hidden
63 self.mbr = args.mbr
64
65 self.lineno = lineno
66 self.source_file = ""
67
68 def get_extra_block_count(self, current_blocks):
69 """
70 The --size param is reflected in self.size (in kB), and we already
71 have current_blocks (1k) blocks, calculate and return the
72 number of (1k) blocks we need to add to get to --size, 0 if
73 we're already there or beyond.
74 """
75 logger.debug("Requested partition size for %s: %d",
76 self.mountpoint, self.size)
77
78 if not self.size:
79 return 0
80
81 requested_blocks = self.size
82
83 logger.debug("Requested blocks %d, current_blocks %d",
84 requested_blocks, current_blocks)
85
86 if requested_blocks > current_blocks:
87 return requested_blocks - current_blocks
88 else:
89 return 0
90
91 def get_rootfs_size(self, actual_rootfs_size=0):
92 """
93 Calculate the required size of rootfs taking into consideration
94 --size/--fixed-size flags as well as overhead and extra space, as
95 specified in kickstart file. Raises an error if the
96 `actual_rootfs_size` is larger than fixed-size rootfs.
97
98 """
99 if self.fixed_size:
100 rootfs_size = self.fixed_size
101 if actual_rootfs_size > rootfs_size:
102 raise WicError("Actual rootfs size (%d kB) is larger than "
103 "allowed size %d kB" %
104 (actual_rootfs_size, rootfs_size))
105 else:
106 extra_blocks = self.get_extra_block_count(actual_rootfs_size)
107 if extra_blocks < self.extra_space:
108 extra_blocks = self.extra_space
109
110 rootfs_size = actual_rootfs_size + extra_blocks
111 rootfs_size = int(rootfs_size * self.overhead_factor)
112
113 logger.debug("Added %d extra blocks to %s to get to %d total blocks",
114 extra_blocks, self.mountpoint, rootfs_size)
115
116 return rootfs_size
117
118 @property
119 def disk_size(self):
120 """
121 Obtain on-disk size of partition taking into consideration
122 --size/--fixed-size options.
123
124 """
125 return self.fixed_size if self.fixed_size else self.size
126
127 def prepare(self, creator, cr_workdir, oe_builddir, rootfs_dir,
128 bootimg_dir, kernel_dir, native_sysroot, updated_fstab_path):
129 """
130 Prepare content for individual partitions, depending on
131 partition command parameters.
132 """
133 self.updated_fstab_path = updated_fstab_path
134 if self.updated_fstab_path and not (self.fstype.startswith("ext") or self.fstype == "msdos"):
135 self.update_fstab_in_rootfs = True
136
137 if not self.source:
138 if self.fstype == "none" or self.no_table:
139 return
140 if not self.size and not self.fixed_size:
141 raise WicError("The %s partition has a size of zero. Please "
142 "specify a non-zero --size/--fixed-size for that "
143 "partition." % self.mountpoint)
144
145 if self.fstype == "swap":
146 self.prepare_swap_partition(cr_workdir, oe_builddir,
147 native_sysroot)
148 self.source_file = "%s/fs.%s" % (cr_workdir, self.fstype)
149 else:
150 if self.fstype in ('squashfs', 'erofs'):
151 raise WicError("It's not possible to create empty %s "
152 "partition '%s'" % (self.fstype, self.mountpoint))
153
154 rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label,
155 self.lineno, self.fstype)
156 if os.path.isfile(rootfs):
157 os.remove(rootfs)
158
159 prefix = "ext" if self.fstype.startswith("ext") else self.fstype
160 method = getattr(self, "prepare_empty_partition_" + prefix)
161 method(rootfs, oe_builddir, native_sysroot)
162 self.source_file = rootfs
163 return
164
165 plugins = PluginMgr.get_plugins('source')
166
167 if self.source not in plugins:
168 raise WicError("The '%s' --source specified for %s doesn't exist.\n\t"
169 "See 'wic list source-plugins' for a list of available"
170 " --sources.\n\tSee 'wic help source-plugins' for "
171 "details on adding a new source plugin." %
172 (self.source, self.mountpoint))
173
174 srcparams_dict = {}
175 if self.sourceparams:
176 # Split sourceparams string of the form key1=val1[,key2=val2,...]
177 # into a dict. Also accepts valueless keys i.e. without =
178 splitted = self.sourceparams.split(',')
179 srcparams_dict = dict((par.split('=', 1) + [None])[:2] for par in splitted if par)
180
181 plugin = PluginMgr.get_plugins('source')[self.source]
182 plugin.do_configure_partition(self, srcparams_dict, creator,
183 cr_workdir, oe_builddir, bootimg_dir,
184 kernel_dir, native_sysroot)
185 plugin.do_stage_partition(self, srcparams_dict, creator,
186 cr_workdir, oe_builddir, bootimg_dir,
187 kernel_dir, native_sysroot)
188 plugin.do_prepare_partition(self, srcparams_dict, creator,
189 cr_workdir, oe_builddir, bootimg_dir,
190 kernel_dir, rootfs_dir, native_sysroot)
191 plugin.do_post_partition(self, srcparams_dict, creator,
192 cr_workdir, oe_builddir, bootimg_dir,
193 kernel_dir, rootfs_dir, native_sysroot)
194
195 # further processing required Partition.size to be an integer, make
196 # sure that it is one
197 if not isinstance(self.size, int):
198 raise WicError("Partition %s internal size is not an integer. "
199 "This a bug in source plugin %s and needs to be fixed." %
200 (self.mountpoint, self.source))
201
202 if self.fixed_size and self.size > self.fixed_size:
203 raise WicError("File system image of partition %s is "
204 "larger (%d kB) than its allowed size %d kB" %
205 (self.mountpoint, self.size, self.fixed_size))
206
207 def prepare_rootfs(self, cr_workdir, oe_builddir, rootfs_dir,
208 native_sysroot, real_rootfs = True, pseudo_dir = None):
209 """
210 Prepare content for a rootfs partition i.e. create a partition
211 and fill it from a /rootfs dir.
212
213 Currently handles ext2/3/4, btrfs, vfat and squashfs.
214 """
215
216 rootfs = "%s/rootfs_%s.%s.%s" % (cr_workdir, self.label,
217 self.lineno, self.fstype)
218 if os.path.isfile(rootfs):
219 os.remove(rootfs)
220
221 p_prefix = os.environ.get("PSEUDO_PREFIX", "%s/usr" % native_sysroot)
222 if (pseudo_dir):
223 # Canonicalize the ignore paths. This corresponds to
224 # calling oe.path.canonicalize(), which is used in bitbake.conf.
225 ignore_paths = [rootfs] + (get_bitbake_var("PSEUDO_IGNORE_PATHS") or "").split(",")
226 canonical_paths = []
227 for path in ignore_paths:
228 if "$" not in path:
229 trailing_slash = path.endswith("/") and "/" or ""
230 canonical_paths.append(os.path.realpath(path) + trailing_slash)
231 ignore_paths = ",".join(canonical_paths)
232
233 pseudo = "export PSEUDO_PREFIX=%s;" % p_prefix
234 pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % pseudo_dir
235 pseudo += "export PSEUDO_PASSWD=%s;" % rootfs_dir
236 pseudo += "export PSEUDO_NOSYMLINKEXP=1;"
237 pseudo += "export PSEUDO_IGNORE_PATHS=%s;" % ignore_paths
238 pseudo += "%s " % get_bitbake_var("FAKEROOTCMD")
239 else:
240 pseudo = None
241
242 if not self.size and real_rootfs:
243 # The rootfs size is not set in .ks file so try to get it
244 # from bitbake variable
245 rsize_bb = get_bitbake_var('ROOTFS_SIZE')
246 rdir = get_bitbake_var('IMAGE_ROOTFS')
247 if rsize_bb and rdir == rootfs_dir:
248 # Bitbake variable ROOTFS_SIZE is calculated in
249 # Image._get_rootfs_size method from meta/lib/oe/image.py
250 # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT,
251 # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE
252 self.size = int(round(float(rsize_bb)))
253 else:
254 # Bitbake variable ROOTFS_SIZE is not defined so compute it
255 # from the rootfs_dir size using the same logic found in
256 # get_rootfs_size() from meta/classes/image.bbclass
257 du_cmd = "du -ks %s" % rootfs_dir
258 out = exec_cmd(du_cmd)
259 self.size = int(out.split()[0])
260
261 prefix = "ext" if self.fstype.startswith("ext") else self.fstype
262 method = getattr(self, "prepare_rootfs_" + prefix)
263 method(rootfs, cr_workdir, oe_builddir, rootfs_dir, native_sysroot, pseudo)
264 self.source_file = rootfs
265
266 # get the rootfs size in the right units for kickstart (kB)
267 du_cmd = "du -Lbks %s" % rootfs
268 out = exec_cmd(du_cmd)
269 self.size = int(out.split()[0])
270
271 def prepare_rootfs_ext(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
272 native_sysroot, pseudo):
273 """
274 Prepare content for an ext2/3/4 rootfs partition.
275 """
276 du_cmd = "du -ks %s" % rootfs_dir
277 out = exec_cmd(du_cmd)
278 actual_rootfs_size = int(out.split()[0])
279
280 rootfs_size = self.get_rootfs_size(actual_rootfs_size)
281
282 with open(rootfs, 'w') as sparse:
283 os.ftruncate(sparse.fileno(), rootfs_size * 1024)
284
285 extraopts = self.mkfs_extraopts or "-F -i 8192"
286
287 # use hash_seed to generate reproducible ext4 images
288 (extraopts, pseudo) = self.get_hash_seed_ext4(extraopts, pseudo)
289
290 label_str = ""
291 if self.label:
292 label_str = "-L %s" % self.label
293
294 mkfs_cmd = "mkfs.%s %s %s %s -U %s -d %s" % \
295 (self.fstype, extraopts, rootfs, label_str, self.fsuuid, rootfs_dir)
296 exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
297
298 if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
299 debugfs_script_path = os.path.join(cr_workdir, "debugfs_script")
300 with open(debugfs_script_path, "w") as f:
301 f.write("cd etc\n")
302 f.write("rm fstab\n")
303 f.write("write %s fstab\n" % (self.updated_fstab_path))
304 debugfs_cmd = "debugfs -w -f %s %s" % (debugfs_script_path, rootfs)
305 exec_native_cmd(debugfs_cmd, native_sysroot)
306
307 mkfs_cmd = "fsck.%s -pvfD %s" % (self.fstype, rootfs)
308 exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
309
310 if os.getenv('SOURCE_DATE_EPOCH'):
311 sde_time = hex(int(os.getenv('SOURCE_DATE_EPOCH')))
312 debugfs_script_path = os.path.join(cr_workdir, "debugfs_script")
313 files = []
314 for root, dirs, others in os.walk(rootfs_dir):
315 base = root.replace(rootfs_dir, "").rstrip(os.sep)
316 files += [ "/" if base == "" else base ]
317 files += [ base + "/" + n for n in dirs + others ]
318 with open(debugfs_script_path, "w") as f:
319 f.write("set_current_time %s\n" % (sde_time))
320 if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
321 f.write("set_inode_field /etc/fstab mtime %s\n" % (sde_time))
322 f.write("set_inode_field /etc/fstab mtime_extra 0\n")
323 for file in set(files):
324 for time in ["atime", "ctime", "crtime"]:
325 f.write("set_inode_field \"%s\" %s %s\n" % (file, time, sde_time))
326 f.write("set_inode_field \"%s\" %s_extra 0\n" % (file, time))
327 for time in ["wtime", "mkfs_time", "lastcheck"]:
328 f.write("set_super_value %s %s\n" % (time, sde_time))
329 for time in ["mtime", "first_error_time", "last_error_time"]:
330 f.write("set_super_value %s 0\n" % (time))
331 debugfs_cmd = "debugfs -w -f %s %s" % (debugfs_script_path, rootfs)
332 exec_native_cmd(debugfs_cmd, native_sysroot)
333
334 self.check_for_Y2038_problem(rootfs, native_sysroot)
335
336 def get_hash_seed_ext4(self, extraopts, pseudo):
337 if os.getenv('SOURCE_DATE_EPOCH'):
338 sde_time = int(os.getenv('SOURCE_DATE_EPOCH'))
339 if pseudo:
340 pseudo = "export E2FSPROGS_FAKE_TIME=%s;%s " % (sde_time, pseudo)
341 else:
342 pseudo = "export E2FSPROGS_FAKE_TIME=%s; " % sde_time
343
344 # Set hash_seed to generate deterministic directory indexes
345 namespace = uuid.UUID("e7429877-e7b3-4a68-a5c9-2f2fdf33d460")
346 if self.fsuuid:
347 namespace = uuid.UUID(self.fsuuid)
348 hash_seed = str(uuid.uuid5(namespace, str(sde_time)))
349 extraopts += " -E hash_seed=%s" % hash_seed
350
351 return (extraopts, pseudo)
352
353 def prepare_rootfs_btrfs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
354 native_sysroot, pseudo):
355 """
356 Prepare content for a btrfs rootfs partition.
357 """
358 du_cmd = "du -ks %s" % rootfs_dir
359 out = exec_cmd(du_cmd)
360 actual_rootfs_size = int(out.split()[0])
361
362 rootfs_size = self.get_rootfs_size(actual_rootfs_size)
363
364 with open(rootfs, 'w') as sparse:
365 os.ftruncate(sparse.fileno(), rootfs_size * 1024)
366
367 label_str = ""
368 if self.label:
369 label_str = "-L %s" % self.label
370
371 mkfs_cmd = "mkfs.%s -b %d -r %s %s %s -U %s %s" % \
372 (self.fstype, rootfs_size * 1024, rootfs_dir, label_str,
373 self.mkfs_extraopts, self.fsuuid, rootfs)
374 exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
375
376 def prepare_rootfs_msdos(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
377 native_sysroot, pseudo):
378 """
379 Prepare content for a msdos/vfat rootfs partition.
380 """
381 du_cmd = "du -bks %s" % rootfs_dir
382 out = exec_cmd(du_cmd)
383 blocks = int(out.split()[0])
384
385 rootfs_size = self.get_rootfs_size(blocks)
386
387 label_str = "-n boot"
388 if self.label:
389 label_str = "-n %s" % self.label
390
391 size_str = ""
392
393 extraopts = self.mkfs_extraopts or '-S 512'
394
395 dosfs_cmd = "mkdosfs %s -i %s %s %s -C %s %d" % \
396 (label_str, self.fsuuid, size_str, extraopts, rootfs,
397 rootfs_size)
398 exec_native_cmd(dosfs_cmd, native_sysroot)
399
400 mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, rootfs_dir)
401 exec_native_cmd(mcopy_cmd, native_sysroot)
402
403 if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
404 mcopy_cmd = "mcopy -m -i %s %s ::/etc/fstab" % (rootfs, self.updated_fstab_path)
405 exec_native_cmd(mcopy_cmd, native_sysroot)
406
407 chmod_cmd = "chmod 644 %s" % rootfs
408 exec_cmd(chmod_cmd)
409
410 prepare_rootfs_vfat = prepare_rootfs_msdos
411
412 def prepare_rootfs_squashfs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
413 native_sysroot, pseudo):
414 """
415 Prepare content for a squashfs rootfs partition.
416 """
417 extraopts = self.mkfs_extraopts or '-noappend'
418 squashfs_cmd = "mksquashfs %s %s %s" % \
419 (rootfs_dir, rootfs, extraopts)
420 exec_native_cmd(squashfs_cmd, native_sysroot, pseudo=pseudo)
421
422 def prepare_rootfs_erofs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
423 native_sysroot, pseudo):
424 """
425 Prepare content for a erofs rootfs partition.
426 """
427 extraopts = self.mkfs_extraopts or ''
428 erofs_cmd = "mkfs.erofs %s -U %s %s %s" % \
429 (extraopts, self.fsuuid, rootfs, rootfs_dir)
430 exec_native_cmd(erofs_cmd, native_sysroot, pseudo=pseudo)
431
432 def prepare_empty_partition_none(self, rootfs, oe_builddir, native_sysroot):
433 pass
434
435 def prepare_empty_partition_ext(self, rootfs, oe_builddir,
436 native_sysroot):
437 """
438 Prepare an empty ext2/3/4 partition.
439 """
440 size = self.disk_size
441 with open(rootfs, 'w') as sparse:
442 os.ftruncate(sparse.fileno(), size * 1024)
443
444 extraopts = self.mkfs_extraopts or "-i 8192"
445
446 # use hash_seed to generate reproducible ext4 images
447 (extraopts, pseudo) = self.get_hash_seed_ext4(extraopts, None)
448
449 label_str = ""
450 if self.label:
451 label_str = "-L %s" % self.label
452
453 mkfs_cmd = "mkfs.%s -F %s %s -U %s %s" % \
454 (self.fstype, extraopts, label_str, self.fsuuid, rootfs)
455 exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
456
457 self.check_for_Y2038_problem(rootfs, native_sysroot)
458
459 def prepare_empty_partition_btrfs(self, rootfs, oe_builddir,
460 native_sysroot):
461 """
462 Prepare an empty btrfs partition.
463 """
464 size = self.disk_size
465 with open(rootfs, 'w') as sparse:
466 os.ftruncate(sparse.fileno(), size * 1024)
467
468 label_str = ""
469 if self.label:
470 label_str = "-L %s" % self.label
471
472 mkfs_cmd = "mkfs.%s -b %d %s -U %s %s %s" % \
473 (self.fstype, self.size * 1024, label_str, self.fsuuid,
474 self.mkfs_extraopts, rootfs)
475 exec_native_cmd(mkfs_cmd, native_sysroot)
476
477 def prepare_empty_partition_msdos(self, rootfs, oe_builddir,
478 native_sysroot):
479 """
480 Prepare an empty vfat partition.
481 """
482 blocks = self.disk_size
483
484 label_str = "-n boot"
485 if self.label:
486 label_str = "-n %s" % self.label
487
488 size_str = ""
489
490 extraopts = self.mkfs_extraopts or '-S 512'
491
492 dosfs_cmd = "mkdosfs %s -i %s %s %s -C %s %d" % \
493 (label_str, self.fsuuid, extraopts, size_str, rootfs,
494 blocks)
495
496 exec_native_cmd(dosfs_cmd, native_sysroot)
497
498 chmod_cmd = "chmod 644 %s" % rootfs
499 exec_cmd(chmod_cmd)
500
501 prepare_empty_partition_vfat = prepare_empty_partition_msdos
502
503 def prepare_swap_partition(self, cr_workdir, oe_builddir, native_sysroot):
504 """
505 Prepare a swap partition.
506 """
507 path = "%s/fs.%s" % (cr_workdir, self.fstype)
508
509 with open(path, 'w') as sparse:
510 os.ftruncate(sparse.fileno(), self.size * 1024)
511
512 label_str = ""
513 if self.label:
514 label_str = "-L %s" % self.label
515
516 mkswap_cmd = "mkswap %s -U %s %s" % (label_str, self.fsuuid, path)
517 exec_native_cmd(mkswap_cmd, native_sysroot)
518
519 def check_for_Y2038_problem(self, rootfs, native_sysroot):
520 """
521 Check if the filesystem is affected by the Y2038 problem
522 (Y2038 problem = 32 bit time_t overflow in January 2038)
523 """
524 def get_err_str(part):
525 err = "The {} filesystem {} has no Y2038 support."
526 if part.mountpoint:
527 args = [part.fstype, "mounted at %s" % part.mountpoint]
528 elif part.label:
529 args = [part.fstype, "labeled '%s'" % part.label]
530 elif part.part_name:
531 args = [part.fstype, "in partition '%s'" % part.part_name]
532 else:
533 args = [part.fstype, "in partition %s" % part.num]
534 return err.format(*args)
535
536 # ext2 and ext3 are always affected by the Y2038 problem
537 if self.fstype in ["ext2", "ext3"]:
538 logger.warn(get_err_str(self))
539 return
540
541 ret, out = exec_native_cmd("dumpe2fs %s" % rootfs, native_sysroot)
542
543 # if ext4 is affected by the Y2038 problem depends on the inode size
544 for line in out.splitlines():
545 if line.startswith("Inode size:"):
546 size = int(line.split(":")[1].strip())
547 if size < 256:
548 logger.warn("%s Inodes (of size %d) are too small." %
549 (get_err_str(self), size))
550 break
551
diff --git a/meta-xilinx-core/scripts/lib/wic/pluginbase.py b/meta-xilinx-core/scripts/lib/wic/pluginbase.py
new file mode 100644
index 00000000..b6456833
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/pluginbase.py
@@ -0,0 +1,144 @@
1#!/usr/bin/env python3
2#
3# Copyright (c) 2011 Intel, Inc.
4#
5# SPDX-License-Identifier: GPL-2.0-only
6#
7
8__all__ = ['ImagerPlugin', 'SourcePlugin']
9
10import os
11import logging
12import types
13
14from collections import defaultdict
15import importlib
16import importlib.util
17
18from wic import WicError
19from wic.misc import get_bitbake_var
20
21PLUGIN_TYPES = ["imager", "source"]
22
23SCRIPTS_PLUGIN_DIR = ["scripts/lib/wic/plugins", "lib/wic/plugins"]
24
25logger = logging.getLogger('wic')
26
27PLUGINS = defaultdict(dict)
28
29class PluginMgr:
30 _plugin_dirs = []
31
32 @classmethod
33 def get_plugins(cls, ptype):
34 """Get dictionary of <plugin_name>:<class> pairs."""
35 if ptype not in PLUGIN_TYPES:
36 raise WicError('%s is not valid plugin type' % ptype)
37
38 # collect plugin directories
39 if not cls._plugin_dirs:
40 cls._plugin_dirs = [os.path.join(os.path.dirname(__file__), 'plugins')]
41 layers = get_bitbake_var("BBLAYERS") or ''
42 for layer_path in layers.split():
43 for script_plugin_dir in SCRIPTS_PLUGIN_DIR:
44 path = os.path.join(layer_path, script_plugin_dir)
45 path = os.path.abspath(os.path.expanduser(path))
46 if path not in cls._plugin_dirs and os.path.isdir(path):
47 cls._plugin_dirs.insert(0, path)
48
49 if ptype not in PLUGINS:
50 # load all ptype plugins
51 for pdir in cls._plugin_dirs:
52 ppath = os.path.join(pdir, ptype)
53 if os.path.isdir(ppath):
54 for fname in os.listdir(ppath):
55 if fname.endswith('.py'):
56 mname = fname[:-3]
57 mpath = os.path.join(ppath, fname)
58 logger.debug("loading plugin module %s", mpath)
59 spec = importlib.util.spec_from_file_location(mname, mpath)
60 module = importlib.util.module_from_spec(spec)
61 spec.loader.exec_module(module)
62
63 return PLUGINS.get(ptype)
64
65class PluginMeta(type):
66 def __new__(cls, name, bases, attrs):
67 class_type = type.__new__(cls, name, bases, attrs)
68 if 'name' in attrs:
69 PLUGINS[class_type.wic_plugin_type][attrs['name']] = class_type
70
71 return class_type
72
73class ImagerPlugin(metaclass=PluginMeta):
74 wic_plugin_type = "imager"
75
76 def do_create(self):
77 raise WicError("Method %s.do_create is not implemented" %
78 self.__class__.__name__)
79
80class SourcePlugin(metaclass=PluginMeta):
81 wic_plugin_type = "source"
82 """
83 The methods that can be implemented by --source plugins.
84
85 Any methods not implemented in a subclass inherit these.
86 """
87
88 @classmethod
89 def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir,
90 bootimg_dir, kernel_dir, native_sysroot):
91 """
92 Called after all partitions have been prepared and assembled into a
93 disk image. This provides a hook to allow finalization of a
94 disk image e.g. to write an MBR to it.
95 """
96 logger.debug("SourcePlugin: do_install_disk: disk: %s", disk_name)
97
98 @classmethod
99 def do_stage_partition(cls, part, source_params, creator, cr_workdir,
100 oe_builddir, bootimg_dir, kernel_dir,
101 native_sysroot):
102 """
103 Special content staging hook called before do_prepare_partition(),
104 normally empty.
105
106 Typically, a partition will just use the passed-in parame e.g
107 straight bootimg_dir, etc, but in some cases, things need to
108 be more tailored e.g. to use a deploy dir + /boot, etc. This
109 hook allows those files to be staged in a customized fashion.
110 Not that get_bitbake_var() allows you to acces non-standard
111 variables that you might want to use for this.
112 """
113 logger.debug("SourcePlugin: do_stage_partition: part: %s", part)
114
115 @classmethod
116 def do_configure_partition(cls, part, source_params, creator, cr_workdir,
117 oe_builddir, bootimg_dir, kernel_dir,
118 native_sysroot):
119 """
120 Called before do_prepare_partition(), typically used to create
121 custom configuration files for a partition, for example
122 syslinux or grub config files.
123 """
124 logger.debug("SourcePlugin: do_configure_partition: part: %s", part)
125
126 @classmethod
127 def do_prepare_partition(cls, part, source_params, creator, cr_workdir,
128 oe_builddir, bootimg_dir, kernel_dir, rootfs_dir,
129 native_sysroot):
130 """
131 Called to do the actual content population for a partition i.e. it
132 'prepares' the partition to be incorporated into the image.
133 """
134 logger.debug("SourcePlugin: do_prepare_partition: part: %s", part)
135
136 @classmethod
137 def do_post_partition(cls, part, source_params, creator, cr_workdir,
138 oe_builddir, bootimg_dir, kernel_dir, rootfs_dir,
139 native_sysroot):
140 """
141 Called after the partition is created. It is useful to add post
142 operations e.g. security signing the partition.
143 """
144 logger.debug("SourcePlugin: do_post_partition: part: %s", part)
diff --git a/meta-xilinx-core/scripts/lib/wic/plugins/imager/direct.py b/meta-xilinx-core/scripts/lib/wic/plugins/imager/direct.py
new file mode 100644
index 00000000..a1d15265
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/plugins/imager/direct.py
@@ -0,0 +1,694 @@
1#
2# Copyright (c) 2013, Intel Corporation.
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6# DESCRIPTION
7# This implements the 'direct' imager plugin class for 'wic'
8#
9# AUTHORS
10# Tom Zanussi <tom.zanussi (at] linux.intel.com>
11#
12
13import logging
14import os
15import random
16import shutil
17import tempfile
18import uuid
19
20from time import strftime
21
22from oe.path import copyhardlinktree
23
24from wic import WicError
25from wic.filemap import sparse_copy
26from wic.ksparser import KickStart, KickStartError
27from wic.pluginbase import PluginMgr, ImagerPlugin
28from wic.misc import get_bitbake_var, exec_cmd, exec_native_cmd
29
30logger = logging.getLogger('wic')
31
32class DirectPlugin(ImagerPlugin):
33 """
34 Install a system into a file containing a partitioned disk image.
35
36 An image file is formatted with a partition table, each partition
37 created from a rootfs or other OpenEmbedded build artifact and dd'ed
38 into the virtual disk. The disk image can subsequently be dd'ed onto
39 media and used on actual hardware.
40 """
41 name = 'direct'
42
43 def __init__(self, wks_file, rootfs_dir, bootimg_dir, kernel_dir,
44 native_sysroot, oe_builddir, options):
45 try:
46 self.ks = KickStart(wks_file)
47 except KickStartError as err:
48 raise WicError(str(err))
49
50 # parse possible 'rootfs=name' items
51 self.rootfs_dir = dict(rdir.split('=') for rdir in rootfs_dir.split(' '))
52 self.bootimg_dir = bootimg_dir
53 self.kernel_dir = kernel_dir
54 self.native_sysroot = native_sysroot
55 self.oe_builddir = oe_builddir
56
57 self.debug = options.debug
58 self.outdir = options.outdir
59 self.compressor = options.compressor
60 self.bmap = options.bmap
61 self.no_fstab_update = options.no_fstab_update
62 self.updated_fstab_path = None
63
64 self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0],
65 strftime("%Y%m%d%H%M"))
66 self.workdir = self.setup_workdir(options.workdir)
67 self._image = None
68 self.ptable_format = self.ks.bootloader.ptable
69 self.parts = self.ks.partitions
70
71 # as a convenience, set source to the boot partition source
72 # instead of forcing it to be set via bootloader --source
73 for part in self.parts:
74 if not self.ks.bootloader.source and part.mountpoint == "/boot":
75 self.ks.bootloader.source = part.source
76 break
77
78 image_path = self._full_path(self.workdir, self.parts[0].disk, "direct")
79 self._image = PartitionedImage(image_path, self.ptable_format,
80 self.parts, self.native_sysroot,
81 options.extra_space)
82
83 def setup_workdir(self, workdir):
84 if workdir:
85 if os.path.exists(workdir):
86 raise WicError("Internal workdir '%s' specified in wic arguments already exists!" % (workdir))
87
88 os.makedirs(workdir)
89 return workdir
90 else:
91 return tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.')
92
93 def do_create(self):
94 """
95 Plugin entry point.
96 """
97 try:
98 self.create()
99 self.assemble()
100 self.finalize()
101 self.print_info()
102 finally:
103 self.cleanup()
104
105 def update_fstab(self, image_rootfs):
106 """Assume partition order same as in wks"""
107 if not image_rootfs:
108 return
109
110 fstab_path = image_rootfs + "/etc/fstab"
111 if not os.path.isfile(fstab_path):
112 return
113
114 with open(fstab_path) as fstab:
115 fstab_lines = fstab.readlines()
116
117 updated = False
118 for part in self.parts:
119 if not part.realnum or not part.mountpoint \
120 or part.mountpoint == "/" or not (part.mountpoint.startswith('/') or part.mountpoint == "swap"):
121 continue
122
123 if part.use_uuid:
124 if part.fsuuid:
125 # FAT UUID is different from others
126 if len(part.fsuuid) == 10:
127 device_name = "UUID=%s-%s" % \
128 (part.fsuuid[2:6], part.fsuuid[6:])
129 else:
130 device_name = "UUID=%s" % part.fsuuid
131 else:
132 device_name = "PARTUUID=%s" % part.uuid
133 elif part.use_label:
134 device_name = "LABEL=%s" % part.label
135 else:
136 # mmc device partitions are named mmcblk0p1, mmcblk0p2..
137 prefix = 'p' if part.disk.startswith('mmcblk') else ''
138 device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum)
139
140 opts = part.fsopts if part.fsopts else "defaults"
141 passno = part.fspassno if part.fspassno else "0"
142 line = "\t".join([device_name, part.mountpoint, part.fstype,
143 opts, "0", passno]) + "\n"
144
145 fstab_lines.append(line)
146 updated = True
147
148 if updated:
149 self.updated_fstab_path = os.path.join(self.workdir, "fstab")
150 with open(self.updated_fstab_path, "w") as f:
151 f.writelines(fstab_lines)
152 if os.getenv('SOURCE_DATE_EPOCH'):
153 fstab_time = int(os.getenv('SOURCE_DATE_EPOCH'))
154 os.utime(self.updated_fstab_path, (fstab_time, fstab_time))
155
156 def _full_path(self, path, name, extention):
157 """ Construct full file path to a file we generate. """
158 return os.path.join(path, "%s-%s.%s" % (self.name, name, extention))
159
160 #
161 # Actual implemention
162 #
163 def create(self):
164 """
165 For 'wic', we already have our build artifacts - we just create
166 filesystems from the artifacts directly and combine them into
167 a partitioned image.
168 """
169 if not self.no_fstab_update:
170 self.update_fstab(self.rootfs_dir.get("ROOTFS_DIR"))
171
172 for part in self.parts:
173 # get rootfs size from bitbake variable if it's not set in .ks file
174 if not part.size:
175 # and if rootfs name is specified for the partition
176 image_name = self.rootfs_dir.get(part.rootfs_dir)
177 if image_name and os.path.sep not in image_name:
178 # Bitbake variable ROOTFS_SIZE is calculated in
179 # Image._get_rootfs_size method from meta/lib/oe/image.py
180 # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT,
181 # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE
182 rsize_bb = get_bitbake_var('ROOTFS_SIZE', image_name)
183 if rsize_bb:
184 part.size = int(round(float(rsize_bb)))
185
186 self._image.prepare(self)
187 self._image.layout_partitions()
188 self._image.create()
189
190 def assemble(self):
191 """
192 Assemble partitions into disk image
193 """
194 self._image.assemble()
195
196 def finalize(self):
197 """
198 Finalize the disk image.
199
200 For example, prepare the image to be bootable by e.g.
201 creating and installing a bootloader configuration.
202 """
203 source_plugin = self.ks.bootloader.source
204 disk_name = self.parts[0].disk
205 if source_plugin:
206 plugin = PluginMgr.get_plugins('source')[source_plugin]
207 plugin.do_install_disk(self._image, disk_name, self, self.workdir,
208 self.oe_builddir, self.bootimg_dir,
209 self.kernel_dir, self.native_sysroot)
210
211 full_path = self._image.path
212 # Generate .bmap
213 if self.bmap:
214 logger.debug("Generating bmap file for %s", disk_name)
215 python = os.path.join(self.native_sysroot, 'usr/bin/python3-native/python3')
216 bmaptool = os.path.join(self.native_sysroot, 'usr/bin/bmaptool')
217 exec_native_cmd("%s %s create %s -o %s.bmap" % \
218 (python, bmaptool, full_path, full_path), self.native_sysroot)
219 # Compress the image
220 if self.compressor:
221 logger.debug("Compressing disk %s with %s", disk_name, self.compressor)
222 exec_cmd("%s %s" % (self.compressor, full_path))
223
224 def print_info(self):
225 """
226 Print the image(s) and artifacts used, for the user.
227 """
228 msg = "The new image(s) can be found here:\n"
229
230 extension = "direct" + {"gzip": ".gz",
231 "bzip2": ".bz2",
232 "xz": ".xz",
233 None: ""}.get(self.compressor)
234 full_path = self._full_path(self.outdir, self.parts[0].disk, extension)
235 msg += ' %s\n\n' % full_path
236
237 msg += 'The following build artifacts were used to create the image(s):\n'
238 for part in self.parts:
239 if part.rootfs_dir is None:
240 continue
241 if part.mountpoint == '/':
242 suffix = ':'
243 else:
244 suffix = '["%s"]:' % (part.mountpoint or part.label)
245 rootdir = part.rootfs_dir
246 msg += ' ROOTFS_DIR%s%s\n' % (suffix.ljust(20), rootdir)
247
248 msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir
249 msg += ' KERNEL_DIR: %s\n' % self.kernel_dir
250 msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot
251
252 logger.info(msg)
253
254 @property
255 def rootdev(self):
256 """
257 Get root device name to use as a 'root' parameter
258 in kernel command line.
259
260 Assume partition order same as in wks
261 """
262 for part in self.parts:
263 if part.mountpoint == "/":
264 if part.uuid:
265 return "PARTUUID=%s" % part.uuid
266 elif part.label and self.ptable_format != 'msdos':
267 return "PARTLABEL=%s" % part.label
268 else:
269 suffix = 'p' if part.disk.startswith('mmcblk') else ''
270 return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum)
271
272 def cleanup(self):
273 if self._image:
274 self._image.cleanup()
275
276 # Move results to the output dir
277 if not os.path.exists(self.outdir):
278 os.makedirs(self.outdir)
279
280 for fname in os.listdir(self.workdir):
281 path = os.path.join(self.workdir, fname)
282 if os.path.isfile(path):
283 shutil.move(path, os.path.join(self.outdir, fname))
284
285 # remove work directory when it is not in debugging mode
286 if not self.debug:
287 shutil.rmtree(self.workdir, ignore_errors=True)
288
289# Overhead of the MBR partitioning scheme (just one sector)
290MBR_OVERHEAD = 1
291
292# Overhead of the GPT partitioning scheme
293GPT_OVERHEAD = 34
294
295# Size of a sector in bytes
296SECTOR_SIZE = 512
297
298class PartitionedImage():
299 """
300 Partitioned image in a file.
301 """
302
303 def __init__(self, path, ptable_format, partitions, native_sysroot=None, extra_space=0):
304 self.path = path # Path to the image file
305 self.numpart = 0 # Number of allocated partitions
306 self.realpart = 0 # Number of partitions in the partition table
307 self.primary_part_num = 0 # Number of primary partitions (msdos)
308 self.extendedpart = 0 # Create extended partition before this logical partition (msdos)
309 self.extended_size_sec = 0 # Size of exteded partition (msdos)
310 self.logical_part_cnt = 0 # Number of total logical paritions (msdos)
311 self.offset = 0 # Offset of next partition (in sectors)
312 self.min_size = 0 # Minimum required disk size to fit
313 # all partitions (in bytes)
314 self.ptable_format = ptable_format # Partition table format
315 # Disk system identifier
316 if os.getenv('SOURCE_DATE_EPOCH'):
317 self.identifier = random.Random(int(os.getenv('SOURCE_DATE_EPOCH'))).randint(1, 0xffffffff)
318 else:
319 self.identifier = random.SystemRandom().randint(1, 0xffffffff)
320
321 self.partitions = partitions
322 self.partimages = []
323 # Size of a sector used in calculations
324 self.sector_size = SECTOR_SIZE
325 self.native_sysroot = native_sysroot
326 num_real_partitions = len([p for p in self.partitions if not p.no_table])
327 self.extra_space = extra_space
328
329 # calculate the real partition number, accounting for partitions not
330 # in the partition table and logical partitions
331 realnum = 0
332 for part in self.partitions:
333 if part.no_table:
334 part.realnum = 0
335 else:
336 realnum += 1
337 if self.ptable_format == 'msdos' and realnum > 3 and num_real_partitions > 4:
338 part.realnum = realnum + 1
339 continue
340 part.realnum = realnum
341
342 # generate parition and filesystem UUIDs
343 for part in self.partitions:
344 if not part.uuid and part.use_uuid:
345 if self.ptable_format in ('gpt', 'gpt-hybrid'):
346 part.uuid = str(uuid.uuid4())
347 else: # msdos partition table
348 part.uuid = '%08x-%02d' % (self.identifier, part.realnum)
349 if not part.fsuuid:
350 if part.fstype == 'vfat' or part.fstype == 'msdos':
351 part.fsuuid = '0x' + str(uuid.uuid4())[:8].upper()
352 else:
353 part.fsuuid = str(uuid.uuid4())
354 else:
355 #make sure the fsuuid for vfat/msdos align with format 0xYYYYYYYY
356 if part.fstype == 'vfat' or part.fstype == 'msdos':
357 if part.fsuuid.upper().startswith("0X"):
358 part.fsuuid = '0x' + part.fsuuid.upper()[2:].rjust(8,"0")
359 else:
360 part.fsuuid = '0x' + part.fsuuid.upper().rjust(8,"0")
361
362 def prepare(self, imager):
363 """Prepare an image. Call prepare method of all image partitions."""
364 for part in self.partitions:
365 # need to create the filesystems in order to get their
366 # sizes before we can add them and do the layout.
367 part.prepare(imager, imager.workdir, imager.oe_builddir,
368 imager.rootfs_dir, imager.bootimg_dir,
369 imager.kernel_dir, imager.native_sysroot,
370 imager.updated_fstab_path)
371
372 # Converting kB to sectors for parted
373 part.size_sec = part.disk_size * 1024 // self.sector_size
374
375 def layout_partitions(self):
376 """ Layout the partitions, meaning calculate the position of every
377 partition on the disk. The 'ptable_format' parameter defines the
378 partition table format and may be "msdos". """
379
380 logger.debug("Assigning %s partitions to disks", self.ptable_format)
381
382 # The number of primary and logical partitions. Extended partition and
383 # partitions not listed in the table are not included.
384 num_real_partitions = len([p for p in self.partitions if not p.no_table])
385
386 # Go through partitions in the order they are added in .ks file
387 for num in range(len(self.partitions)):
388 part = self.partitions[num]
389
390 if self.ptable_format == 'msdos' and part.part_name:
391 raise WicError("setting custom partition name is not " \
392 "implemented for msdos partitions")
393
394 if self.ptable_format == 'msdos' and part.part_type:
395 # The --part-type can also be implemented for MBR partitions,
396 # in which case it would map to the 1-byte "partition type"
397 # filed at offset 3 of the partition entry.
398 raise WicError("setting custom partition type is not " \
399 "implemented for msdos partitions")
400
401 if part.mbr and self.ptable_format != 'gpt-hybrid':
402 raise WicError("Partition may only be included in MBR with " \
403 "a gpt-hybrid partition table")
404
405 # Get the disk where the partition is located
406 self.numpart += 1
407 if not part.no_table:
408 self.realpart += 1
409
410 if self.numpart == 1:
411 if self.ptable_format == "msdos":
412 overhead = MBR_OVERHEAD
413 elif self.ptable_format in ("gpt", "gpt-hybrid"):
414 overhead = GPT_OVERHEAD
415
416 # Skip one sector required for the partitioning scheme overhead
417 self.offset += overhead
418
419 if self.ptable_format == "msdos":
420 if self.primary_part_num > 3 or \
421 (self.extendedpart == 0 and self.primary_part_num >= 3 and num_real_partitions > 4):
422 part.type = 'logical'
423 # Reserve a sector for EBR for every logical partition
424 # before alignment is performed.
425 if part.type == 'logical':
426 self.offset += 2
427
428 align_sectors = 0
429 if part.align:
430 # If not first partition and we do have alignment set we need
431 # to align the partition.
432 # FIXME: This leaves a empty spaces to the disk. To fill the
433 # gaps we could enlargea the previous partition?
434
435 # Calc how much the alignment is off.
436 align_sectors = self.offset % (part.align * 1024 // self.sector_size)
437
438 if align_sectors:
439 # If partition is not aligned as required, we need
440 # to move forward to the next alignment point
441 align_sectors = (part.align * 1024 // self.sector_size) - align_sectors
442
443 logger.debug("Realignment for %s%s with %s sectors, original"
444 " offset %s, target alignment is %sK.",
445 part.disk, self.numpart, align_sectors,
446 self.offset, part.align)
447
448 # increase the offset so we actually start the partition on right alignment
449 self.offset += align_sectors
450
451 if part.offset is not None:
452 offset = part.offset // self.sector_size
453
454 if offset * self.sector_size != part.offset:
455 raise WicError("Could not place %s%s at offset %d with sector size %d" % (part.disk, self.numpart, part.offset, self.sector_size))
456
457 delta = offset - self.offset
458 if delta < 0:
459 raise WicError("Could not place %s%s at offset %d: next free sector is %d (delta: %d)" % (part.disk, self.numpart, part.offset, self.offset, delta))
460
461 logger.debug("Skipping %d sectors to place %s%s at offset %dK",
462 delta, part.disk, self.numpart, part.offset)
463
464 self.offset = offset
465
466 part.start = self.offset
467 self.offset += part.size_sec
468
469 if not part.no_table:
470 part.num = self.realpart
471 else:
472 part.num = 0
473
474 if self.ptable_format == "msdos" and not part.no_table:
475 if part.type == 'logical':
476 self.logical_part_cnt += 1
477 part.num = self.logical_part_cnt + 4
478 if self.extendedpart == 0:
479 # Create extended partition as a primary partition
480 self.primary_part_num += 1
481 self.extendedpart = part.num
482 else:
483 self.extended_size_sec += align_sectors
484 self.extended_size_sec += part.size_sec + 2
485 else:
486 self.primary_part_num += 1
487 part.num = self.primary_part_num
488
489 logger.debug("Assigned %s to %s%d, sectors range %d-%d size %d "
490 "sectors (%d bytes).", part.mountpoint, part.disk,
491 part.num, part.start, self.offset - 1, part.size_sec,
492 part.size_sec * self.sector_size)
493
494 # Once all the partitions have been layed out, we can calculate the
495 # minumim disk size
496 self.min_size = self.offset
497 if self.ptable_format in ("gpt", "gpt-hybrid"):
498 self.min_size += GPT_OVERHEAD
499
500 self.min_size *= self.sector_size
501 self.min_size += self.extra_space
502
503 def _create_partition(self, device, parttype, fstype, start, size):
504 """ Create a partition on an image described by the 'device' object. """
505
506 # Start is included to the size so we need to substract one from the end.
507 end = start + size - 1
508 logger.debug("Added '%s' partition, sectors %d-%d, size %d sectors",
509 parttype, start, end, size)
510
511 cmd = "parted -s %s unit s mkpart %s" % (device, parttype)
512 if fstype:
513 cmd += " %s" % fstype
514 cmd += " %d %d" % (start, end)
515
516 return exec_native_cmd(cmd, self.native_sysroot)
517
518 def _write_identifier(self, device, identifier):
519 logger.debug("Set disk identifier %x", identifier)
520 with open(device, 'r+b') as img:
521 img.seek(0x1B8)
522 img.write(identifier.to_bytes(4, 'little'))
523
524 def _make_disk(self, device, ptable_format, min_size):
525 logger.debug("Creating sparse file %s", device)
526 with open(device, 'w') as sparse:
527 os.ftruncate(sparse.fileno(), min_size)
528
529 logger.debug("Initializing partition table for %s", device)
530 exec_native_cmd("parted -s %s mklabel %s" % (device, ptable_format),
531 self.native_sysroot)
532
533 def _write_disk_guid(self):
534 if self.ptable_format in ('gpt', 'gpt-hybrid'):
535 if os.getenv('SOURCE_DATE_EPOCH'):
536 self.disk_guid = uuid.UUID(int=int(os.getenv('SOURCE_DATE_EPOCH')))
537 else:
538 self.disk_guid = uuid.uuid4()
539
540 logger.debug("Set disk guid %s", self.disk_guid)
541 sfdisk_cmd = "sfdisk --disk-id %s %s" % (self.path, self.disk_guid)
542 exec_native_cmd(sfdisk_cmd, self.native_sysroot)
543
544 def create(self):
545 self._make_disk(self.path,
546 "gpt" if self.ptable_format == "gpt-hybrid" else self.ptable_format,
547 self.min_size)
548
549 self._write_identifier(self.path, self.identifier)
550 self._write_disk_guid()
551
552 if self.ptable_format == "gpt-hybrid":
553 mbr_path = self.path + ".mbr"
554 self._make_disk(mbr_path, "msdos", self.min_size)
555 self._write_identifier(mbr_path, self.identifier)
556
557 logger.debug("Creating partitions")
558
559 hybrid_mbr_part_num = 0
560
561 for part in self.partitions:
562 if part.num == 0:
563 continue
564
565 if self.ptable_format == "msdos" and part.num == self.extendedpart:
566 # Create an extended partition (note: extended
567 # partition is described in MBR and contains all
568 # logical partitions). The logical partitions save a
569 # sector for an EBR just before the start of a
570 # partition. The extended partition must start one
571 # sector before the start of the first logical
572 # partition. This way the first EBR is inside of the
573 # extended partition. Since the extended partitions
574 # starts a sector before the first logical partition,
575 # add a sector at the back, so that there is enough
576 # room for all logical partitions.
577 self._create_partition(self.path, "extended",
578 None, part.start - 2,
579 self.extended_size_sec)
580
581 if part.fstype == "swap":
582 parted_fs_type = "linux-swap"
583 elif part.fstype == "vfat":
584 parted_fs_type = "fat32"
585 elif part.fstype == "msdos":
586 parted_fs_type = "fat16"
587 if not part.system_id:
588 part.system_id = '0x6' # FAT16
589 else:
590 # Type for ext2/ext3/ext4/btrfs
591 parted_fs_type = "ext2"
592
593 # Boot ROM of OMAP boards require vfat boot partition to have an
594 # even number of sectors.
595 if part.mountpoint == "/boot" and part.fstype in ["vfat", "msdos"] \
596 and part.size_sec % 2:
597 logger.debug("Subtracting one sector from '%s' partition to "
598 "get even number of sectors for the partition",
599 part.mountpoint)
600 part.size_sec -= 1
601
602 self._create_partition(self.path, part.type,
603 parted_fs_type, part.start, part.size_sec)
604
605 if self.ptable_format == "gpt-hybrid" and part.mbr:
606 hybrid_mbr_part_num += 1
607 if hybrid_mbr_part_num > 4:
608 raise WicError("Extended MBR partitions are not supported in hybrid MBR")
609 self._create_partition(mbr_path, "primary",
610 parted_fs_type, part.start, part.size_sec)
611
612 if self.ptable_format in ("gpt", "gpt-hybrid") and (part.part_name or part.label):
613 partition_label = part.part_name if part.part_name else part.label
614 logger.debug("partition %d: set name to %s",
615 part.num, partition_label)
616 exec_native_cmd("sgdisk --change-name=%d:%s %s" % \
617 (part.num, partition_label,
618 self.path), self.native_sysroot)
619
620 if part.part_type:
621 logger.debug("partition %d: set type UID to %s",
622 part.num, part.part_type)
623 exec_native_cmd("sgdisk --typecode=%d:%s %s" % \
624 (part.num, part.part_type,
625 self.path), self.native_sysroot)
626
627 if part.uuid and self.ptable_format in ("gpt", "gpt-hybrid"):
628 logger.debug("partition %d: set UUID to %s",
629 part.num, part.uuid)
630 exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \
631 (part.num, part.uuid, self.path),
632 self.native_sysroot)
633
634 if part.active:
635 flag_name = "legacy_boot" if self.ptable_format in ('gpt', 'gpt-hybrid') else "boot"
636 logger.debug("Set '%s' flag for partition '%s' on disk '%s'",
637 flag_name, part.num, self.path)
638 exec_native_cmd("parted -s %s set %d %s on" % \
639 (self.path, part.num, flag_name),
640 self.native_sysroot)
641 if self.ptable_format == 'gpt-hybrid' and part.mbr:
642 exec_native_cmd("parted -s %s set %d %s on" % \
643 (mbr_path, hybrid_mbr_part_num, "boot"),
644 self.native_sysroot)
645 if part.system_id:
646 exec_native_cmd("sfdisk --part-type %s %s %s" % \
647 (self.path, part.num, part.system_id),
648 self.native_sysroot)
649
650 if part.hidden and self.ptable_format == "gpt":
651 logger.debug("Set hidden attribute for partition '%s' on disk '%s'",
652 part.num, self.path)
653 exec_native_cmd("sfdisk --part-attrs %s %s RequiredPartition" % \
654 (self.path, part.num),
655 self.native_sysroot)
656
657 if self.ptable_format == "gpt-hybrid":
658 # Write a protective GPT partition
659 hybrid_mbr_part_num += 1
660 if hybrid_mbr_part_num > 4:
661 raise WicError("Extended MBR partitions are not supported in hybrid MBR")
662
663 # parted cannot directly create a protective GPT partition, so
664 # create with an arbitrary type, then change it to the correct type
665 # with sfdisk
666 self._create_partition(mbr_path, "primary", "fat32", 1, GPT_OVERHEAD)
667 exec_native_cmd("sfdisk --part-type %s %d 0xee" % (mbr_path, hybrid_mbr_part_num),
668 self.native_sysroot)
669
670 # Copy hybrid MBR
671 with open(mbr_path, "rb") as mbr_file:
672 with open(self.path, "r+b") as image_file:
673 mbr = mbr_file.read(512)
674 image_file.write(mbr)
675
676 def cleanup(self):
677 pass
678
679 def assemble(self):
680 logger.debug("Installing partitions")
681
682 for part in self.partitions:
683 source = part.source_file
684 if source:
685 # install source_file contents into a partition
686 sparse_copy(source, self.path, seek=part.start * self.sector_size)
687
688 logger.debug("Installed %s in partition %d, sectors %d-%d, "
689 "size %d sectors", source, part.num, part.start,
690 part.start + part.size_sec - 1, part.size_sec)
691
692 partimage = self.path + '.p%d' % part.num
693 os.rename(source, partimage)
694 self.partimages.append(partimage)
diff --git a/meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-biosplusefi.py b/meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-biosplusefi.py
new file mode 100644
index 00000000..5bd73906
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-biosplusefi.py
@@ -0,0 +1,213 @@
1#
2# This program is free software; you can redistribute it and/or modify
3# it under the terms of the GNU General Public License version 2 as
4# published by the Free Software Foundation.
5#
6# This program is distributed in the hope that it will be useful,
7# but WITHOUT ANY WARRANTY; without even the implied warranty of
8# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9# GNU General Public License for more details.
10#
11# You should have received a copy of the GNU General Public License along
12# with this program; if not, write to the Free Software Foundation, Inc.,
13# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
14#
15# DESCRIPTION
16# This implements the 'bootimg-biosplusefi' source plugin class for 'wic'
17#
18# AUTHORS
19# William Bourque <wbourque [at) gmail.com>
20
21import types
22
23from wic.pluginbase import SourcePlugin
24from importlib.machinery import SourceFileLoader
25
26class BootimgBiosPlusEFIPlugin(SourcePlugin):
27 """
28 Create MBR + EFI boot partition
29
30 This plugin creates a boot partition that contains both
31 legacy BIOS and EFI content. It will be able to boot from both.
32 This is useful when managing PC fleet with some older machines
33 without EFI support.
34
35 Note it is possible to create an image that can boot from both
36 legacy BIOS and EFI by defining two partitions : one with arg
37 --source bootimg-efi and another one with --source bootimg-pcbios.
38 However, this method has the obvious downside that it requires TWO
39 partitions to be created on the storage device.
40 Both partitions will also be marked as "bootable" which does not work on
41 most BIOS, has BIOS often uses the "bootable" flag to determine
42 what to boot. If you have such a BIOS, you need to manually remove the
43 "bootable" flag from the EFI partition for the drive to be bootable.
44 Having two partitions also seems to confuse wic : the content of
45 the first partition will be duplicated into the second, even though it
46 will not be used at all.
47
48 Also, unlike "isoimage-isohybrid" that also does BIOS and EFI, this plugin
49 allows you to have more than only a single rootfs partitions and does
50 not turn the rootfs into an initramfs RAM image.
51
52 This plugin is made to put everything into a single /boot partition so it
53 does not have the limitations listed above.
54
55 The plugin is made so it does tries not to reimplement what's already
56 been done in other plugins; as such it imports "bootimg-pcbios"
57 and "bootimg-efi".
58 Plugin "bootimg-pcbios" is used to generate legacy BIOS boot.
59 Plugin "bootimg-efi" is used to generate the UEFI boot. Note that it
60 requires a --sourceparams argument to know which loader to use; refer
61 to "bootimg-efi" code/documentation for the list of loader.
62
63 Imports are handled with "SourceFileLoader" from importlib as it is
64 otherwise very difficult to import module that has hyphen "-" in their
65 filename.
66 The SourcePlugin() methods used in the plugins (do_install_disk,
67 do_configure_partition, do_prepare_partition) are then called on both,
68 beginning by "bootimg-efi".
69
70 Plugin options, such as "--sourceparams" can still be passed to a
71 plugin, as long they does not cause issue in the other plugin.
72
73 Example wic configuration:
74 part /boot --source bootimg-biosplusefi --sourceparams="loader=grub-efi"\\
75 --ondisk sda --label os_boot --active --align 1024 --use-uuid
76 """
77
78 name = 'bootimg-biosplusefi'
79
80 __PCBIOS_MODULE_NAME = "bootimg-pcbios"
81 __EFI_MODULE_NAME = "bootimg-efi"
82
83 __imgEFIObj = None
84 __imgBiosObj = None
85
86 @classmethod
87 def __init__(cls):
88 """
89 Constructor (init)
90 """
91
92 # XXX
93 # For some reasons, __init__ constructor is never called.
94 # Something to do with how pluginbase works?
95 cls.__instanciateSubClasses()
96
97 @classmethod
98 def __instanciateSubClasses(cls):
99 """
100
101 """
102
103 # Import bootimg-pcbios (class name "BootimgPcbiosPlugin")
104 modulePath = os.path.join(os.path.dirname(os.path.realpath(__file__)),
105 cls.__PCBIOS_MODULE_NAME + ".py")
106 loader = SourceFileLoader(cls.__PCBIOS_MODULE_NAME, modulePath)
107 mod = types.ModuleType(loader.name)
108 loader.exec_module(mod)
109 cls.__imgBiosObj = mod.BootimgPcbiosPlugin()
110
111 # Import bootimg-efi (class name "BootimgEFIPlugin")
112 modulePath = os.path.join(os.path.dirname(os.path.realpath(__file__)),
113 cls.__EFI_MODULE_NAME + ".py")
114 loader = SourceFileLoader(cls.__EFI_MODULE_NAME, modulePath)
115 mod = types.ModuleType(loader.name)
116 loader.exec_module(mod)
117 cls.__imgEFIObj = mod.BootimgEFIPlugin()
118
119 @classmethod
120 def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir,
121 bootimg_dir, kernel_dir, native_sysroot):
122 """
123 Called after all partitions have been prepared and assembled into a
124 disk image.
125 """
126
127 if ( (not cls.__imgEFIObj) or (not cls.__imgBiosObj) ):
128 cls.__instanciateSubClasses()
129
130 cls.__imgEFIObj.do_install_disk(
131 disk,
132 disk_name,
133 creator,
134 workdir,
135 oe_builddir,
136 bootimg_dir,
137 kernel_dir,
138 native_sysroot)
139
140 cls.__imgBiosObj.do_install_disk(
141 disk,
142 disk_name,
143 creator,
144 workdir,
145 oe_builddir,
146 bootimg_dir,
147 kernel_dir,
148 native_sysroot)
149
150 @classmethod
151 def do_configure_partition(cls, part, source_params, creator, cr_workdir,
152 oe_builddir, bootimg_dir, kernel_dir,
153 native_sysroot):
154 """
155 Called before do_prepare_partition()
156 """
157
158 if ( (not cls.__imgEFIObj) or (not cls.__imgBiosObj) ):
159 cls.__instanciateSubClasses()
160
161 cls.__imgEFIObj.do_configure_partition(
162 part,
163 source_params,
164 creator,
165 cr_workdir,
166 oe_builddir,
167 bootimg_dir,
168 kernel_dir,
169 native_sysroot)
170
171 cls.__imgBiosObj.do_configure_partition(
172 part,
173 source_params,
174 creator,
175 cr_workdir,
176 oe_builddir,
177 bootimg_dir,
178 kernel_dir,
179 native_sysroot)
180
181 @classmethod
182 def do_prepare_partition(cls, part, source_params, creator, cr_workdir,
183 oe_builddir, bootimg_dir, kernel_dir,
184 rootfs_dir, native_sysroot):
185 """
186 Called to do the actual content population for a partition i.e. it
187 'prepares' the partition to be incorporated into the image.
188 """
189
190 if ( (not cls.__imgEFIObj) or (not cls.__imgBiosObj) ):
191 cls.__instanciateSubClasses()
192
193 cls.__imgEFIObj.do_prepare_partition(
194 part,
195 source_params,
196 creator,
197 cr_workdir,
198 oe_builddir,
199 bootimg_dir,
200 kernel_dir,
201 rootfs_dir,
202 native_sysroot)
203
204 cls.__imgBiosObj.do_prepare_partition(
205 part,
206 source_params,
207 creator,
208 cr_workdir,
209 oe_builddir,
210 bootimg_dir,
211 kernel_dir,
212 rootfs_dir,
213 native_sysroot)
diff --git a/meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-efi.py b/meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-efi.py
new file mode 100644
index 00000000..7cc51315
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-efi.py
@@ -0,0 +1,507 @@
1#
2# Copyright (c) 2014, Intel Corporation.
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6# DESCRIPTION
7# This implements the 'bootimg-efi' source plugin class for 'wic'
8#
9# AUTHORS
10# Tom Zanussi <tom.zanussi (at] linux.intel.com>
11#
12
13import logging
14import os
15import tempfile
16import shutil
17import re
18
19from glob import glob
20
21from wic import WicError
22from wic.engine import get_custom_config
23from wic.pluginbase import SourcePlugin
24from wic.misc import (exec_cmd, exec_native_cmd,
25 get_bitbake_var, BOOTDD_EXTRA_SPACE)
26
27logger = logging.getLogger('wic')
28
29class BootimgEFIPlugin(SourcePlugin):
30 """
31 Create EFI boot partition.
32 This plugin supports GRUB 2 and systemd-boot bootloaders.
33 """
34
35 name = 'bootimg-efi'
36
37 @classmethod
38 def _copy_additional_files(cls, hdddir, initrd, dtb):
39 bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
40 if not bootimg_dir:
41 raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
42
43 if initrd:
44 initrds = initrd.split(';')
45 for rd in initrds:
46 cp_cmd = "cp %s/%s %s" % (bootimg_dir, rd, hdddir)
47 exec_cmd(cp_cmd, True)
48 else:
49 logger.debug("Ignoring missing initrd")
50
51 if dtb:
52 if ';' in dtb:
53 raise WicError("Only one DTB supported, exiting")
54 cp_cmd = "cp %s/%s %s" % (bootimg_dir, dtb, hdddir)
55 exec_cmd(cp_cmd, True)
56
57 @classmethod
58 def do_configure_grubefi(cls, hdddir, creator, cr_workdir, source_params):
59 """
60 Create loader-specific (grub-efi) config
61 """
62 configfile = creator.ks.bootloader.configfile
63 custom_cfg = None
64 if configfile:
65 custom_cfg = get_custom_config(configfile)
66 if custom_cfg:
67 # Use a custom configuration for grub
68 grubefi_conf = custom_cfg
69 logger.debug("Using custom configuration file "
70 "%s for grub.cfg", configfile)
71 else:
72 raise WicError("configfile is specified but failed to "
73 "get it from %s." % configfile)
74
75 initrd = source_params.get('initrd')
76 dtb = source_params.get('dtb')
77
78 cls._copy_additional_files(hdddir, initrd, dtb)
79
80 if not custom_cfg:
81 # Create grub configuration using parameters from wks file
82 bootloader = creator.ks.bootloader
83 title = source_params.get('title')
84
85 grubefi_conf = ""
86 grubefi_conf += "serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1\n"
87 grubefi_conf += "default=boot\n"
88 grubefi_conf += "timeout=%s\n" % bootloader.timeout
89 grubefi_conf += "menuentry '%s'{\n" % (title if title else "boot")
90
91 kernel = get_bitbake_var("KERNEL_IMAGETYPE")
92 if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1":
93 if get_bitbake_var("INITRAMFS_IMAGE"):
94 kernel = "%s-%s.bin" % \
95 (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
96
97 label = source_params.get('label')
98 label_conf = "root=%s" % creator.rootdev
99 if label:
100 label_conf = "LABEL=%s" % label
101
102 grubefi_conf += "linux /%s %s rootwait %s\n" \
103 % (kernel, label_conf, bootloader.append)
104
105 if initrd:
106 initrds = initrd.split(';')
107 grubefi_conf += "initrd"
108 for rd in initrds:
109 grubefi_conf += " /%s" % rd
110 grubefi_conf += "\n"
111
112 if dtb:
113 grubefi_conf += "devicetree /%s\n" % dtb
114
115 grubefi_conf += "}\n"
116
117 logger.debug("Writing grubefi config %s/hdd/boot/EFI/BOOT/grub.cfg",
118 cr_workdir)
119 cfg = open("%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir, "w")
120 cfg.write(grubefi_conf)
121 cfg.close()
122
123 @classmethod
124 def do_configure_systemdboot(cls, hdddir, creator, cr_workdir, source_params):
125 """
126 Create loader-specific systemd-boot/gummiboot config
127 """
128 install_cmd = "install -d %s/loader" % hdddir
129 exec_cmd(install_cmd)
130
131 install_cmd = "install -d %s/loader/entries" % hdddir
132 exec_cmd(install_cmd)
133
134 bootloader = creator.ks.bootloader
135
136 unified_image = source_params.get('create-unified-kernel-image') == "true"
137
138 loader_conf = ""
139 if not unified_image:
140 loader_conf += "default boot\n"
141 loader_conf += "timeout %d\n" % bootloader.timeout
142
143 initrd = source_params.get('initrd')
144 dtb = source_params.get('dtb')
145
146 if not unified_image:
147 cls._copy_additional_files(hdddir, initrd, dtb)
148
149 logger.debug("Writing systemd-boot config "
150 "%s/hdd/boot/loader/loader.conf", cr_workdir)
151 cfg = open("%s/hdd/boot/loader/loader.conf" % cr_workdir, "w")
152 cfg.write(loader_conf)
153 cfg.close()
154
155 configfile = creator.ks.bootloader.configfile
156 custom_cfg = None
157 if configfile:
158 custom_cfg = get_custom_config(configfile)
159 if custom_cfg:
160 # Use a custom configuration for systemd-boot
161 boot_conf = custom_cfg
162 logger.debug("Using custom configuration file "
163 "%s for systemd-boots's boot.conf", configfile)
164 else:
165 raise WicError("configfile is specified but failed to "
166 "get it from %s.", configfile)
167
168 if not custom_cfg:
169 # Create systemd-boot configuration using parameters from wks file
170 kernel = get_bitbake_var("KERNEL_IMAGETYPE")
171 if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1":
172 if get_bitbake_var("INITRAMFS_IMAGE"):
173 kernel = "%s-%s.bin" % \
174 (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
175
176 title = source_params.get('title')
177
178 boot_conf = ""
179 boot_conf += "title %s\n" % (title if title else "boot")
180 boot_conf += "linux /%s\n" % kernel
181
182 label = source_params.get('label')
183 label_conf = "LABEL=Boot root=%s" % creator.rootdev
184 if label:
185 label_conf = "LABEL=%s" % label
186
187 boot_conf += "options %s %s\n" % \
188 (label_conf, bootloader.append)
189
190 if initrd:
191 initrds = initrd.split(';')
192 for rd in initrds:
193 boot_conf += "initrd /%s\n" % rd
194
195 if dtb:
196 boot_conf += "devicetree /%s\n" % dtb
197
198 if not unified_image:
199 logger.debug("Writing systemd-boot config "
200 "%s/hdd/boot/loader/entries/boot.conf", cr_workdir)
201 cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w")
202 cfg.write(boot_conf)
203 cfg.close()
204
205
206 @classmethod
207 def do_configure_partition(cls, part, source_params, creator, cr_workdir,
208 oe_builddir, bootimg_dir, kernel_dir,
209 native_sysroot):
210 """
211 Called before do_prepare_partition(), creates loader-specific config
212 """
213 hdddir = "%s/hdd/boot" % cr_workdir
214
215 install_cmd = "install -d %s/EFI/BOOT" % hdddir
216 exec_cmd(install_cmd)
217
218 try:
219 if source_params['loader'] == 'grub-efi':
220 cls.do_configure_grubefi(hdddir, creator, cr_workdir, source_params)
221 elif source_params['loader'] == 'systemd-boot':
222 cls.do_configure_systemdboot(hdddir, creator, cr_workdir, source_params)
223 elif source_params['loader'] == 'uefi-kernel':
224 pass
225 else:
226 raise WicError("unrecognized bootimg-efi loader: %s" % source_params['loader'])
227 except KeyError:
228 raise WicError("bootimg-efi requires a loader, none specified")
229
230 if get_bitbake_var("IMAGE_EFI_BOOT_FILES") is None:
231 logger.debug('No boot files defined in IMAGE_EFI_BOOT_FILES')
232 else:
233 boot_files = None
234 for (fmt, id) in (("_uuid-%s", part.uuid), ("_label-%s", part.label), (None, None)):
235 if fmt:
236 var = fmt % id
237 else:
238 var = ""
239
240 boot_files = get_bitbake_var("IMAGE_EFI_BOOT_FILES" + var)
241 if boot_files:
242 break
243
244 logger.debug('Boot files: %s', boot_files)
245
246 # list of tuples (src_name, dst_name)
247 deploy_files = []
248 for src_entry in re.findall(r'[\w;\-\./\*]+', boot_files):
249 if ';' in src_entry:
250 dst_entry = tuple(src_entry.split(';'))
251 if not dst_entry[0] or not dst_entry[1]:
252 raise WicError('Malformed boot file entry: %s' % src_entry)
253 else:
254 dst_entry = (src_entry, src_entry)
255
256 logger.debug('Destination entry: %r', dst_entry)
257 deploy_files.append(dst_entry)
258
259 cls.install_task = [];
260 for deploy_entry in deploy_files:
261 src, dst = deploy_entry
262 if '*' in src:
263 # by default install files under their basename
264 entry_name_fn = os.path.basename
265 if dst != src:
266 # unless a target name was given, then treat name
267 # as a directory and append a basename
268 entry_name_fn = lambda name: \
269 os.path.join(dst,
270 os.path.basename(name))
271
272 srcs = glob(os.path.join(kernel_dir, src))
273
274 logger.debug('Globbed sources: %s', ', '.join(srcs))
275 for entry in srcs:
276 src = os.path.relpath(entry, kernel_dir)
277 entry_dst_name = entry_name_fn(entry)
278 cls.install_task.append((src, entry_dst_name))
279 else:
280 cls.install_task.append((src, dst))
281
282 @classmethod
283 def do_prepare_partition(cls, part, source_params, creator, cr_workdir,
284 oe_builddir, bootimg_dir, kernel_dir,
285 rootfs_dir, native_sysroot):
286 """
287 Called to do the actual content population for a partition i.e. it
288 'prepares' the partition to be incorporated into the image.
289 In this case, prepare content for an EFI (grub) boot partition.
290 """
291 if not kernel_dir:
292 kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
293 if not kernel_dir:
294 raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
295
296 staging_kernel_dir = kernel_dir
297
298 hdddir = "%s/hdd/boot" % cr_workdir
299
300 kernel = get_bitbake_var("KERNEL_IMAGETYPE")
301 if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1":
302 if get_bitbake_var("INITRAMFS_IMAGE"):
303 kernel = "%s-%s.bin" % \
304 (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
305
306 if source_params.get('create-unified-kernel-image') == "true":
307 initrd = source_params.get('initrd')
308 if not initrd:
309 raise WicError("initrd= must be specified when create-unified-kernel-image=true, exiting")
310
311 deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
312 efi_stub = glob("%s/%s" % (deploy_dir, "linux*.efi.stub"))
313 if len(efi_stub) == 0:
314 raise WicError("Unified Kernel Image EFI stub not found, exiting")
315 efi_stub = efi_stub[0]
316
317 with tempfile.TemporaryDirectory() as tmp_dir:
318 label = source_params.get('label')
319 label_conf = "root=%s" % creator.rootdev
320 if label:
321 label_conf = "LABEL=%s" % label
322
323 bootloader = creator.ks.bootloader
324 cmdline = open("%s/cmdline" % tmp_dir, "w")
325 cmdline.write("%s %s" % (label_conf, bootloader.append))
326 cmdline.close()
327
328 initrds = initrd.split(';')
329 initrd = open("%s/initrd" % tmp_dir, "wb")
330 for f in initrds:
331 with open("%s/%s" % (deploy_dir, f), 'rb') as in_file:
332 shutil.copyfileobj(in_file, initrd)
333 initrd.close()
334
335 # Searched by systemd-boot:
336 # https://systemd.io/BOOT_LOADER_SPECIFICATION/#type-2-efi-unified-kernel-images
337 install_cmd = "install -d %s/EFI/Linux" % hdddir
338 exec_cmd(install_cmd)
339
340 staging_dir_host = get_bitbake_var("STAGING_DIR_HOST")
341 target_sys = get_bitbake_var("TARGET_SYS")
342
343 objdump_cmd = "%s-objdump" % target_sys
344 objdump_cmd += " -p %s" % efi_stub
345 objdump_cmd += " | awk '{ if ($1 == \"SectionAlignment\"){print $2} }'"
346
347 ret, align_str = exec_native_cmd(objdump_cmd, native_sysroot)
348 align = int(align_str, 16)
349
350 objdump_cmd = "%s-objdump" % target_sys
351 objdump_cmd += " -h %s | tail -2" % efi_stub
352 ret, output = exec_native_cmd(objdump_cmd, native_sysroot)
353
354 offset = int(output.split()[2], 16) + int(output.split()[3], 16)
355
356 osrel_off = offset + align - offset % align
357 osrel_path = "%s/usr/lib/os-release" % staging_dir_host
358 osrel_sz = os.stat(osrel_path).st_size
359
360 cmdline_off = osrel_off + osrel_sz
361 cmdline_off = cmdline_off + align - cmdline_off % align
362 cmdline_sz = os.stat(cmdline.name).st_size
363
364 dtb_off = cmdline_off + cmdline_sz
365 dtb_off = dtb_off + align - dtb_off % align
366
367 dtb = source_params.get('dtb')
368 if dtb:
369 if ';' in dtb:
370 raise WicError("Only one DTB supported, exiting")
371 dtb_path = "%s/%s" % (deploy_dir, dtb)
372 dtb_params = '--add-section .dtb=%s --change-section-vma .dtb=0x%x' % \
373 (dtb_path, dtb_off)
374 linux_off = dtb_off + os.stat(dtb_path).st_size
375 linux_off = linux_off + align - linux_off % align
376 else:
377 dtb_params = ''
378 linux_off = dtb_off
379
380 linux_path = "%s/%s" % (staging_kernel_dir, kernel)
381 linux_sz = os.stat(linux_path).st_size
382
383 initrd_off = linux_off + linux_sz
384 initrd_off = initrd_off + align - initrd_off % align
385
386 # https://www.freedesktop.org/software/systemd/man/systemd-stub.html
387 objcopy_cmd = "%s-objcopy" % target_sys
388 objcopy_cmd += " --enable-deterministic-archives"
389 objcopy_cmd += " --preserve-dates"
390 objcopy_cmd += " --add-section .osrel=%s" % osrel_path
391 objcopy_cmd += " --change-section-vma .osrel=0x%x" % osrel_off
392 objcopy_cmd += " --add-section .cmdline=%s" % cmdline.name
393 objcopy_cmd += " --change-section-vma .cmdline=0x%x" % cmdline_off
394 objcopy_cmd += dtb_params
395 objcopy_cmd += " --add-section .linux=%s" % linux_path
396 objcopy_cmd += " --change-section-vma .linux=0x%x" % linux_off
397 objcopy_cmd += " --add-section .initrd=%s" % initrd.name
398 objcopy_cmd += " --change-section-vma .initrd=0x%x" % initrd_off
399 objcopy_cmd += " %s %s/EFI/Linux/linux.efi" % (efi_stub, hdddir)
400
401 exec_native_cmd(objcopy_cmd, native_sysroot)
402 else:
403 if source_params.get('install-kernel-into-boot-dir') != 'false':
404 install_cmd = "install -m 0644 %s/%s %s/%s" % \
405 (staging_kernel_dir, kernel, hdddir, kernel)
406 exec_cmd(install_cmd)
407
408 if get_bitbake_var("IMAGE_EFI_BOOT_FILES"):
409 for src_path, dst_path in cls.install_task:
410 install_cmd = "install -m 0644 -D %s %s" \
411 % (os.path.join(kernel_dir, src_path),
412 os.path.join(hdddir, dst_path))
413 exec_cmd(install_cmd)
414
415 try:
416 if source_params['loader'] == 'grub-efi':
417 shutil.copyfile("%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir,
418 "%s/grub.cfg" % cr_workdir)
419 for mod in [x for x in os.listdir(kernel_dir) if x.startswith("grub-efi-")]:
420 cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[9:])
421 exec_cmd(cp_cmd, True)
422 shutil.move("%s/grub.cfg" % cr_workdir,
423 "%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir)
424 elif source_params['loader'] == 'systemd-boot':
425 for mod in [x for x in os.listdir(kernel_dir) if x.startswith("systemd-")]:
426 cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[8:])
427 exec_cmd(cp_cmd, True)
428 elif source_params['loader'] == 'uefi-kernel':
429 kernel = get_bitbake_var("KERNEL_IMAGETYPE")
430 if not kernel:
431 raise WicError("Empty KERNEL_IMAGETYPE")
432 target = get_bitbake_var("TARGET_SYS")
433 if not target:
434 raise WicError("Empty TARGET_SYS")
435
436 if re.match("x86_64", target):
437 kernel_efi_image = "bootx64.efi"
438 elif re.match('i.86', target):
439 kernel_efi_image = "bootia32.efi"
440 elif re.match('aarch64', target):
441 kernel_efi_image = "bootaa64.efi"
442 elif re.match('arm', target):
443 kernel_efi_image = "bootarm.efi"
444 else:
445 raise WicError("UEFI stub kernel is incompatible with target %s" % target)
446
447 for mod in [x for x in os.listdir(kernel_dir) if x.startswith(kernel)]:
448 cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, kernel_efi_image)
449 exec_cmd(cp_cmd, True)
450 else:
451 raise WicError("unrecognized bootimg-efi loader: %s" %
452 source_params['loader'])
453 except KeyError:
454 raise WicError("bootimg-efi requires a loader, none specified")
455
456 startup = os.path.join(kernel_dir, "startup.nsh")
457 if os.path.exists(startup):
458 cp_cmd = "cp %s %s/" % (startup, hdddir)
459 exec_cmd(cp_cmd, True)
460
461 for paths in part.include_path or []:
462 for path in paths:
463 cp_cmd = "cp -r %s %s/" % (path, hdddir)
464 exec_cmd(cp_cmd, True)
465
466 du_cmd = "du -bks %s" % hdddir
467 out = exec_cmd(du_cmd)
468 blocks = int(out.split()[0])
469
470 extra_blocks = part.get_extra_block_count(blocks)
471
472 if extra_blocks < BOOTDD_EXTRA_SPACE:
473 extra_blocks = BOOTDD_EXTRA_SPACE
474
475 blocks += extra_blocks
476
477 logger.debug("Added %d extra blocks to %s to get to %d total blocks",
478 extra_blocks, part.mountpoint, blocks)
479
480 # required for compatibility with certain devices expecting file system
481 # block count to be equal to partition block count
482 if blocks < part.fixed_size:
483 blocks = part.fixed_size
484 logger.debug("Overriding %s to %d total blocks for compatibility",
485 part.mountpoint, blocks)
486
487 # dosfs image, created by mkdosfs
488 bootimg = "%s/boot.img" % cr_workdir
489
490 label = part.label if part.label else "ESP"
491
492 dosfs_cmd = "mkdosfs -n %s -i %s -C %s %d" % \
493 (label, part.fsuuid, bootimg, blocks)
494 exec_native_cmd(dosfs_cmd, native_sysroot)
495
496 mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir)
497 exec_native_cmd(mcopy_cmd, native_sysroot)
498
499 chmod_cmd = "chmod 644 %s" % bootimg
500 exec_cmd(chmod_cmd)
501
502 du_cmd = "du -Lbks %s" % bootimg
503 out = exec_cmd(du_cmd)
504 bootimg_size = out.split()[0]
505
506 part.size = int(bootimg_size)
507 part.source_file = bootimg
diff --git a/meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-partition.py b/meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-partition.py
new file mode 100644
index 00000000..1071d1af
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-partition.py
@@ -0,0 +1,197 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6# DESCRIPTION
7# This implements the 'bootimg-partition' source plugin class for
8# 'wic'. The plugin creates an image of boot partition, copying over
9# files listed in IMAGE_BOOT_FILES bitbake variable.
10#
11# AUTHORS
12# Maciej Borzecki <maciej.borzecki (at] open-rnd.pl>
13#
14
15import logging
16import os
17import re
18
19from glob import glob
20
21from wic import WicError
22from wic.engine import get_custom_config
23from wic.pluginbase import SourcePlugin
24from wic.misc import exec_cmd, get_bitbake_var
25
26logger = logging.getLogger('wic')
27
28class BootimgPartitionPlugin(SourcePlugin):
29 """
30 Create an image of boot partition, copying over files
31 listed in IMAGE_BOOT_FILES bitbake variable.
32 """
33
34 name = 'bootimg-partition'
35 image_boot_files_var_name = 'IMAGE_BOOT_FILES'
36
37 @classmethod
38 def do_configure_partition(cls, part, source_params, cr, cr_workdir,
39 oe_builddir, bootimg_dir, kernel_dir,
40 native_sysroot):
41 """
42 Called before do_prepare_partition(), create u-boot specific boot config
43 """
44 hdddir = "%s/boot.%d" % (cr_workdir, part.lineno)
45 install_cmd = "install -d %s" % hdddir
46 exec_cmd(install_cmd)
47
48 if not kernel_dir:
49 kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
50 if not kernel_dir:
51 raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
52
53 boot_files = None
54 for (fmt, id) in (("_uuid-%s", part.uuid), ("_label-%s", part.label), (None, None)):
55 if fmt:
56 var = fmt % id
57 else:
58 var = ""
59
60 boot_files = get_bitbake_var(cls.image_boot_files_var_name + var)
61 if boot_files is not None:
62 break
63
64 if boot_files is None:
65 raise WicError('No boot files defined, %s unset for entry #%d' % (cls.image_boot_files_var_name, part.lineno))
66
67 logger.debug('Boot files: %s', boot_files)
68
69 # list of tuples (src_name, dst_name)
70 deploy_files = []
71 for src_entry in re.findall(r'[\w;\-\./\*]+', boot_files):
72 if ';' in src_entry:
73 dst_entry = tuple(src_entry.split(';'))
74 if not dst_entry[0] or not dst_entry[1]:
75 raise WicError('Malformed boot file entry: %s' % src_entry)
76 else:
77 dst_entry = (src_entry, src_entry)
78
79 logger.debug('Destination entry: %r', dst_entry)
80 deploy_files.append(dst_entry)
81
82 cls.install_task = [];
83 for deploy_entry in deploy_files:
84 src, dst = deploy_entry
85 if '*' in src:
86 # by default install files under their basename
87 entry_name_fn = os.path.basename
88 if dst != src:
89 # unless a target name was given, then treat name
90 # as a directory and append a basename
91 entry_name_fn = lambda name: \
92 os.path.join(dst,
93 os.path.basename(name))
94
95 srcs = glob(os.path.join(kernel_dir, src))
96
97 logger.debug('Globbed sources: %s', ', '.join(srcs))
98 for entry in srcs:
99 src = os.path.relpath(entry, kernel_dir)
100 entry_dst_name = entry_name_fn(entry)
101 cls.install_task.append((src, entry_dst_name))
102 else:
103 cls.install_task.append((src, dst))
104
105 if source_params.get('loader') != "u-boot":
106 return
107
108 configfile = cr.ks.bootloader.configfile
109 custom_cfg = None
110 if configfile:
111 custom_cfg = get_custom_config(configfile)
112 if custom_cfg:
113 # Use a custom configuration for extlinux.conf
114 extlinux_conf = custom_cfg
115 logger.debug("Using custom configuration file "
116 "%s for extlinux.conf", configfile)
117 else:
118 raise WicError("configfile is specified but failed to "
119 "get it from %s." % configfile)
120
121 if not custom_cfg:
122 # The kernel types supported by the sysboot of u-boot
123 kernel_types = ["zImage", "Image", "fitImage", "uImage", "vmlinux"]
124 has_dtb = False
125 fdt_dir = '/'
126 kernel_name = None
127
128 # Find the kernel image name, from the highest precedence to lowest
129 for image in kernel_types:
130 for task in cls.install_task:
131 src, dst = task
132 if re.match(image, src):
133 kernel_name = os.path.join('/', dst)
134 break
135 if kernel_name:
136 break
137
138 for task in cls.install_task:
139 src, dst = task
140 # We suppose that all the dtb are in the same directory
141 if re.search(r'\.dtb', src) and fdt_dir == '/':
142 has_dtb = True
143 fdt_dir = os.path.join(fdt_dir, os.path.dirname(dst))
144 break
145
146 if not kernel_name:
147 raise WicError('No kernel file found')
148
149 # Compose the extlinux.conf
150 extlinux_conf = "default Yocto\n"
151 extlinux_conf += "label Yocto\n"
152 extlinux_conf += " kernel %s\n" % kernel_name
153 if has_dtb:
154 extlinux_conf += " fdtdir %s\n" % fdt_dir
155 bootloader = cr.ks.bootloader
156 extlinux_conf += "append root=%s rootwait %s\n" \
157 % (cr.rootdev, bootloader.append if bootloader.append else '')
158
159 install_cmd = "install -d %s/extlinux/" % hdddir
160 exec_cmd(install_cmd)
161 cfg = open("%s/extlinux/extlinux.conf" % hdddir, "w")
162 cfg.write(extlinux_conf)
163 cfg.close()
164
165
166 @classmethod
167 def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
168 oe_builddir, bootimg_dir, kernel_dir,
169 rootfs_dir, native_sysroot):
170 """
171 Called to do the actual content population for a partition i.e. it
172 'prepares' the partition to be incorporated into the image.
173 In this case, does the following:
174 - sets up a vfat partition
175 - copies all files listed in IMAGE_BOOT_FILES variable
176 """
177 hdddir = "%s/boot.%d" % (cr_workdir, part.lineno)
178
179 if not kernel_dir:
180 kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
181 if not kernel_dir:
182 raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
183
184 logger.debug('Kernel dir: %s', bootimg_dir)
185
186
187 for task in cls.install_task:
188 src_path, dst_path = task
189 logger.debug('Install %s as %s', src_path, dst_path)
190 install_cmd = "install -m 0644 -D %s %s" \
191 % (os.path.join(kernel_dir, src_path),
192 os.path.join(hdddir, dst_path))
193 exec_cmd(install_cmd)
194
195 logger.debug('Prepare boot partition using rootfs in %s', hdddir)
196 part.prepare_rootfs(cr_workdir, oe_builddir, hdddir,
197 native_sysroot, False)
diff --git a/meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-pcbios.py b/meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-pcbios.py
new file mode 100644
index 00000000..a207a835
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/plugins/source/bootimg-pcbios.py
@@ -0,0 +1,209 @@
1#
2# Copyright (c) 2014, Intel Corporation.
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6# DESCRIPTION
7# This implements the 'bootimg-pcbios' source plugin class for 'wic'
8#
9# AUTHORS
10# Tom Zanussi <tom.zanussi (at] linux.intel.com>
11#
12
13import logging
14import os
15import re
16
17from wic import WicError
18from wic.engine import get_custom_config
19from wic.pluginbase import SourcePlugin
20from wic.misc import (exec_cmd, exec_native_cmd,
21 get_bitbake_var, BOOTDD_EXTRA_SPACE)
22
23logger = logging.getLogger('wic')
24
25class BootimgPcbiosPlugin(SourcePlugin):
26 """
27 Create MBR boot partition and install syslinux on it.
28 """
29
30 name = 'bootimg-pcbios'
31
32 @classmethod
33 def _get_bootimg_dir(cls, bootimg_dir, dirname):
34 """
35 Check if dirname exists in default bootimg_dir or in STAGING_DIR.
36 """
37 staging_datadir = get_bitbake_var("STAGING_DATADIR")
38 for result in (bootimg_dir, staging_datadir):
39 if os.path.exists("%s/%s" % (result, dirname)):
40 return result
41
42 # STAGING_DATADIR is expanded with MLPREFIX if multilib is enabled
43 # but dependency syslinux is still populated to original STAGING_DATADIR
44 nonarch_datadir = re.sub('/[^/]*recipe-sysroot', '/recipe-sysroot', staging_datadir)
45 if os.path.exists(os.path.join(nonarch_datadir, dirname)):
46 return nonarch_datadir
47
48 raise WicError("Couldn't find correct bootimg_dir, exiting")
49
50 @classmethod
51 def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir,
52 bootimg_dir, kernel_dir, native_sysroot):
53 """
54 Called after all partitions have been prepared and assembled into a
55 disk image. In this case, we install the MBR.
56 """
57 bootimg_dir = cls._get_bootimg_dir(bootimg_dir, 'syslinux')
58 mbrfile = "%s/syslinux/" % bootimg_dir
59 if creator.ptable_format == 'msdos':
60 mbrfile += "mbr.bin"
61 elif creator.ptable_format == 'gpt':
62 mbrfile += "gptmbr.bin"
63 else:
64 raise WicError("Unsupported partition table: %s" %
65 creator.ptable_format)
66
67 if not os.path.exists(mbrfile):
68 raise WicError("Couldn't find %s. If using the -e option, do you "
69 "have the right MACHINE set in local.conf? If not, "
70 "is the bootimg_dir path correct?" % mbrfile)
71
72 full_path = creator._full_path(workdir, disk_name, "direct")
73 logger.debug("Installing MBR on disk %s as %s with size %s bytes",
74 disk_name, full_path, disk.min_size)
75
76 dd_cmd = "dd if=%s of=%s conv=notrunc" % (mbrfile, full_path)
77 exec_cmd(dd_cmd, native_sysroot)
78
79 @classmethod
80 def do_configure_partition(cls, part, source_params, creator, cr_workdir,
81 oe_builddir, bootimg_dir, kernel_dir,
82 native_sysroot):
83 """
84 Called before do_prepare_partition(), creates syslinux config
85 """
86 hdddir = "%s/hdd/boot" % cr_workdir
87
88 install_cmd = "install -d %s" % hdddir
89 exec_cmd(install_cmd)
90
91 bootloader = creator.ks.bootloader
92
93 custom_cfg = None
94 if bootloader.configfile:
95 custom_cfg = get_custom_config(bootloader.configfile)
96 if custom_cfg:
97 # Use a custom configuration for grub
98 syslinux_conf = custom_cfg
99 logger.debug("Using custom configuration file %s "
100 "for syslinux.cfg", bootloader.configfile)
101 else:
102 raise WicError("configfile is specified but failed to "
103 "get it from %s." % bootloader.configfile)
104
105 if not custom_cfg:
106 # Create syslinux configuration using parameters from wks file
107 splash = os.path.join(cr_workdir, "/hdd/boot/splash.jpg")
108 if os.path.exists(splash):
109 splashline = "menu background splash.jpg"
110 else:
111 splashline = ""
112
113 syslinux_conf = ""
114 syslinux_conf += "PROMPT 0\n"
115 syslinux_conf += "TIMEOUT " + str(bootloader.timeout) + "\n"
116 syslinux_conf += "\n"
117 syslinux_conf += "ALLOWOPTIONS 1\n"
118 syslinux_conf += "SERIAL 0 115200\n"
119 syslinux_conf += "\n"
120 if splashline:
121 syslinux_conf += "%s\n" % splashline
122 syslinux_conf += "DEFAULT boot\n"
123 syslinux_conf += "LABEL boot\n"
124
125 kernel = "/" + get_bitbake_var("KERNEL_IMAGETYPE")
126 syslinux_conf += "KERNEL " + kernel + "\n"
127
128 syslinux_conf += "APPEND label=boot root=%s %s\n" % \
129 (creator.rootdev, bootloader.append)
130
131 logger.debug("Writing syslinux config %s/hdd/boot/syslinux.cfg",
132 cr_workdir)
133 cfg = open("%s/hdd/boot/syslinux.cfg" % cr_workdir, "w")
134 cfg.write(syslinux_conf)
135 cfg.close()
136
137 @classmethod
138 def do_prepare_partition(cls, part, source_params, creator, cr_workdir,
139 oe_builddir, bootimg_dir, kernel_dir,
140 rootfs_dir, native_sysroot):
141 """
142 Called to do the actual content population for a partition i.e. it
143 'prepares' the partition to be incorporated into the image.
144 In this case, prepare content for legacy bios boot partition.
145 """
146 bootimg_dir = cls._get_bootimg_dir(bootimg_dir, 'syslinux')
147
148 staging_kernel_dir = kernel_dir
149
150 hdddir = "%s/hdd/boot" % cr_workdir
151
152 kernel = get_bitbake_var("KERNEL_IMAGETYPE")
153 if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1":
154 if get_bitbake_var("INITRAMFS_IMAGE"):
155 kernel = "%s-%s.bin" % \
156 (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
157
158 cmds = ("install -m 0644 %s/%s %s/%s" %
159 (staging_kernel_dir, kernel, hdddir, get_bitbake_var("KERNEL_IMAGETYPE")),
160 "install -m 444 %s/syslinux/ldlinux.sys %s/ldlinux.sys" %
161 (bootimg_dir, hdddir),
162 "install -m 0644 %s/syslinux/vesamenu.c32 %s/vesamenu.c32" %
163 (bootimg_dir, hdddir),
164 "install -m 444 %s/syslinux/libcom32.c32 %s/libcom32.c32" %
165 (bootimg_dir, hdddir),
166 "install -m 444 %s/syslinux/libutil.c32 %s/libutil.c32" %
167 (bootimg_dir, hdddir))
168
169 for install_cmd in cmds:
170 exec_cmd(install_cmd)
171
172 du_cmd = "du -bks %s" % hdddir
173 out = exec_cmd(du_cmd)
174 blocks = int(out.split()[0])
175
176 extra_blocks = part.get_extra_block_count(blocks)
177
178 if extra_blocks < BOOTDD_EXTRA_SPACE:
179 extra_blocks = BOOTDD_EXTRA_SPACE
180
181 blocks += extra_blocks
182
183 logger.debug("Added %d extra blocks to %s to get to %d total blocks",
184 extra_blocks, part.mountpoint, blocks)
185
186 # dosfs image, created by mkdosfs
187 bootimg = "%s/boot%s.img" % (cr_workdir, part.lineno)
188
189 label = part.label if part.label else "boot"
190
191 dosfs_cmd = "mkdosfs -n %s -i %s -S 512 -C %s %d" % \
192 (label, part.fsuuid, bootimg, blocks)
193 exec_native_cmd(dosfs_cmd, native_sysroot)
194
195 mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir)
196 exec_native_cmd(mcopy_cmd, native_sysroot)
197
198 syslinux_cmd = "syslinux %s" % bootimg
199 exec_native_cmd(syslinux_cmd, native_sysroot)
200
201 chmod_cmd = "chmod 644 %s" % bootimg
202 exec_cmd(chmod_cmd)
203
204 du_cmd = "du -Lbks %s" % bootimg
205 out = exec_cmd(du_cmd)
206 bootimg_size = out.split()[0]
207
208 part.size = int(bootimg_size)
209 part.source_file = bootimg
diff --git a/meta-xilinx-core/scripts/lib/wic/plugins/source/empty.py b/meta-xilinx-core/scripts/lib/wic/plugins/source/empty.py
new file mode 100644
index 00000000..41789123
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/plugins/source/empty.py
@@ -0,0 +1,89 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: MIT
5#
6
7# The empty wic plugin is used to create unformatted empty partitions for wic
8# images.
9# To use it you must pass "empty" as argument for the "--source" parameter in
10# the wks file. For example:
11# part foo --source empty --ondisk sda --size="1024" --align 1024
12#
13# The plugin supports writing zeros to the start of the
14# partition. This is useful to overwrite old content like
15# filesystem signatures which may be re-recognized otherwise.
16# This feature can be enabled with
17# '--sourceparams="[fill|size=<N>[S|s|K|k|M|G]][,][bs=<N>[S|s|K|k|M|G]]"'
18# Conflicting or missing options throw errors.
19
20import logging
21import os
22
23from wic import WicError
24from wic.ksparser import sizetype
25from wic.pluginbase import SourcePlugin
26
27logger = logging.getLogger('wic')
28
29class EmptyPartitionPlugin(SourcePlugin):
30 """
31 Populate unformatted empty partition.
32
33 The following sourceparams are supported:
34 - fill
35 Fill the entire partition with zeros. Requires '--fixed-size' option
36 to be set.
37 - size=<N>[S|s|K|k|M|G]
38 Set the first N bytes of the partition to zero. Default unit is 'K'.
39 - bs=<N>[S|s|K|k|M|G]
40 Write at most N bytes at a time during source file creation.
41 Defaults to '1M'. Default unit is 'K'.
42 """
43
44 name = 'empty'
45
46 @classmethod
47 def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
48 oe_builddir, bootimg_dir, kernel_dir,
49 rootfs_dir, native_sysroot):
50 """
51 Called to do the actual content population for a partition i.e. it
52 'prepares' the partition to be incorporated into the image.
53 """
54 get_byte_count = sizetype('K', True)
55 size = 0
56
57 if 'fill' in source_params and 'size' in source_params:
58 raise WicError("Conflicting source parameters 'fill' and 'size' specified, exiting.")
59
60 # Set the size of the zeros to be written to the partition
61 if 'fill' in source_params:
62 if part.fixed_size == 0:
63 raise WicError("Source parameter 'fill' only works with the '--fixed-size' option, exiting.")
64 size = get_byte_count(part.fixed_size)
65 elif 'size' in source_params:
66 size = get_byte_count(source_params['size'])
67
68 if size == 0:
69 # Nothing to do, create empty partition
70 return
71
72 if 'bs' in source_params:
73 bs = get_byte_count(source_params['bs'])
74 else:
75 bs = get_byte_count('1M')
76
77 # Create a binary file of the requested size filled with zeros
78 source_file = os.path.join(cr_workdir, 'empty-plugin-zeros%s.bin' % part.lineno)
79 if not os.path.exists(os.path.dirname(source_file)):
80 os.makedirs(os.path.dirname(source_file))
81
82 quotient, remainder = divmod(size, bs)
83 with open(source_file, 'wb') as file:
84 for _ in range(quotient):
85 file.write(bytearray(bs))
86 file.write(bytearray(remainder))
87
88 part.size = (size + 1024 - 1) // 1024 # size in KB rounded up
89 part.source_file = source_file
diff --git a/meta-xilinx-core/scripts/lib/wic/plugins/source/isoimage-isohybrid.py b/meta-xilinx-core/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
new file mode 100644
index 00000000..607356ad
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
@@ -0,0 +1,463 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6# DESCRIPTION
7# This implements the 'isoimage-isohybrid' source plugin class for 'wic'
8#
9# AUTHORS
10# Mihaly Varga <mihaly.varga (at] ni.com>
11
12import glob
13import logging
14import os
15import re
16import shutil
17
18from wic import WicError
19from wic.engine import get_custom_config
20from wic.pluginbase import SourcePlugin
21from wic.misc import exec_cmd, exec_native_cmd, get_bitbake_var
22
23logger = logging.getLogger('wic')
24
25class IsoImagePlugin(SourcePlugin):
26 """
27 Create a bootable ISO image
28
29 This plugin creates a hybrid, legacy and EFI bootable ISO image. The
30 generated image can be used on optical media as well as USB media.
31
32 Legacy boot uses syslinux and EFI boot uses grub or gummiboot (not
33 implemented yet) as bootloader. The plugin creates the directories required
34 by bootloaders and populates them by creating and configuring the
35 bootloader files.
36
37 Example kickstart file:
38 part /boot --source isoimage-isohybrid --sourceparams="loader=grub-efi, \\
39 image_name= IsoImage" --ondisk cd --label LIVECD
40 bootloader --timeout=10 --append=" "
41
42 In --sourceparams "loader" specifies the bootloader used for booting in EFI
43 mode, while "image_name" specifies the name of the generated image. In the
44 example above, wic creates an ISO image named IsoImage-cd.direct (default
45 extension added by direct imeger plugin) and a file named IsoImage-cd.iso
46 """
47
48 name = 'isoimage-isohybrid'
49
50 @classmethod
51 def do_configure_syslinux(cls, creator, cr_workdir):
52 """
53 Create loader-specific (syslinux) config
54 """
55 splash = os.path.join(cr_workdir, "ISO/boot/splash.jpg")
56 if os.path.exists(splash):
57 splashline = "menu background splash.jpg"
58 else:
59 splashline = ""
60
61 bootloader = creator.ks.bootloader
62
63 syslinux_conf = ""
64 syslinux_conf += "PROMPT 0\n"
65 syslinux_conf += "TIMEOUT %s \n" % (bootloader.timeout or 10)
66 syslinux_conf += "\n"
67 syslinux_conf += "ALLOWOPTIONS 1\n"
68 syslinux_conf += "SERIAL 0 115200\n"
69 syslinux_conf += "\n"
70 if splashline:
71 syslinux_conf += "%s\n" % splashline
72 syslinux_conf += "DEFAULT boot\n"
73 syslinux_conf += "LABEL boot\n"
74
75 kernel = get_bitbake_var("KERNEL_IMAGETYPE")
76 if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1":
77 if get_bitbake_var("INITRAMFS_IMAGE"):
78 kernel = "%s-%s.bin" % \
79 (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
80
81 syslinux_conf += "KERNEL /" + kernel + "\n"
82 syslinux_conf += "APPEND initrd=/initrd LABEL=boot %s\n" \
83 % bootloader.append
84
85 logger.debug("Writing syslinux config %s/ISO/isolinux/isolinux.cfg",
86 cr_workdir)
87
88 with open("%s/ISO/isolinux/isolinux.cfg" % cr_workdir, "w") as cfg:
89 cfg.write(syslinux_conf)
90
91 @classmethod
92 def do_configure_grubefi(cls, part, creator, target_dir):
93 """
94 Create loader-specific (grub-efi) config
95 """
96 configfile = creator.ks.bootloader.configfile
97 if configfile:
98 grubefi_conf = get_custom_config(configfile)
99 if grubefi_conf:
100 logger.debug("Using custom configuration file %s for grub.cfg",
101 configfile)
102 else:
103 raise WicError("configfile is specified "
104 "but failed to get it from %s", configfile)
105 else:
106 splash = os.path.join(target_dir, "splash.jpg")
107 if os.path.exists(splash):
108 splashline = "menu background splash.jpg"
109 else:
110 splashline = ""
111
112 bootloader = creator.ks.bootloader
113
114 grubefi_conf = ""
115 grubefi_conf += "serial --unit=0 --speed=115200 --word=8 "
116 grubefi_conf += "--parity=no --stop=1\n"
117 grubefi_conf += "default=boot\n"
118 grubefi_conf += "timeout=%s\n" % (bootloader.timeout or 10)
119 grubefi_conf += "\n"
120 grubefi_conf += "search --set=root --label %s " % part.label
121 grubefi_conf += "\n"
122 grubefi_conf += "menuentry 'boot'{\n"
123
124 kernel = get_bitbake_var("KERNEL_IMAGETYPE")
125 if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1":
126 if get_bitbake_var("INITRAMFS_IMAGE"):
127 kernel = "%s-%s.bin" % \
128 (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
129
130 grubefi_conf += "linux /%s rootwait %s\n" \
131 % (kernel, bootloader.append)
132 grubefi_conf += "initrd /initrd \n"
133 grubefi_conf += "}\n"
134
135 if splashline:
136 grubefi_conf += "%s\n" % splashline
137
138 cfg_path = os.path.join(target_dir, "grub.cfg")
139 logger.debug("Writing grubefi config %s", cfg_path)
140
141 with open(cfg_path, "w") as cfg:
142 cfg.write(grubefi_conf)
143
144 @staticmethod
145 def _build_initramfs_path(rootfs_dir, cr_workdir):
146 """
147 Create path for initramfs image
148 """
149
150 initrd = get_bitbake_var("INITRD_LIVE") or get_bitbake_var("INITRD")
151 if not initrd:
152 initrd_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
153 if not initrd_dir:
154 raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting.")
155
156 image_name = get_bitbake_var("IMAGE_BASENAME")
157 if not image_name:
158 raise WicError("Couldn't find IMAGE_BASENAME, exiting.")
159
160 image_type = get_bitbake_var("INITRAMFS_FSTYPES")
161 if not image_type:
162 raise WicError("Couldn't find INITRAMFS_FSTYPES, exiting.")
163
164 machine = os.path.basename(initrd_dir)
165
166 pattern = '%s/%s*%s.%s' % (initrd_dir, image_name, machine, image_type)
167 files = glob.glob(pattern)
168 if files:
169 initrd = files[0]
170
171 if not initrd or not os.path.exists(initrd):
172 # Create initrd from rootfs directory
173 initrd = "%s/initrd.cpio.gz" % cr_workdir
174 initrd_dir = "%s/INITRD" % cr_workdir
175 shutil.copytree("%s" % rootfs_dir, \
176 "%s" % initrd_dir, symlinks=True)
177
178 if os.path.isfile("%s/init" % rootfs_dir):
179 shutil.copy2("%s/init" % rootfs_dir, "%s/init" % initrd_dir)
180 elif os.path.lexists("%s/init" % rootfs_dir):
181 os.symlink(os.readlink("%s/init" % rootfs_dir), \
182 "%s/init" % initrd_dir)
183 elif os.path.isfile("%s/sbin/init" % rootfs_dir):
184 shutil.copy2("%s/sbin/init" % rootfs_dir, \
185 "%s" % initrd_dir)
186 elif os.path.lexists("%s/sbin/init" % rootfs_dir):
187 os.symlink(os.readlink("%s/sbin/init" % rootfs_dir), \
188 "%s/init" % initrd_dir)
189 else:
190 raise WicError("Couldn't find or build initrd, exiting.")
191
192 exec_cmd("cd %s && find . | cpio -o -H newc -R root:root >%s/initrd.cpio " \
193 % (initrd_dir, cr_workdir), as_shell=True)
194 exec_cmd("gzip -f -9 %s/initrd.cpio" % cr_workdir, as_shell=True)
195 shutil.rmtree(initrd_dir)
196
197 return initrd
198
199 @classmethod
200 def do_configure_partition(cls, part, source_params, creator, cr_workdir,
201 oe_builddir, bootimg_dir, kernel_dir,
202 native_sysroot):
203 """
204 Called before do_prepare_partition(), creates loader-specific config
205 """
206 isodir = "%s/ISO/" % cr_workdir
207
208 if os.path.exists(isodir):
209 shutil.rmtree(isodir)
210
211 install_cmd = "install -d %s " % isodir
212 exec_cmd(install_cmd)
213
214 # Overwrite the name of the created image
215 logger.debug(source_params)
216 if 'image_name' in source_params and \
217 source_params['image_name'].strip():
218 creator.name = source_params['image_name'].strip()
219 logger.debug("The name of the image is: %s", creator.name)
220
221 @staticmethod
222 def _install_payload(source_params, iso_dir):
223 """
224 Copies contents of payload directory (as specified in 'payload_dir' param) into iso_dir
225 """
226
227 if source_params.get('payload_dir'):
228 payload_dir = source_params['payload_dir']
229
230 logger.debug("Payload directory: %s", payload_dir)
231 shutil.copytree(payload_dir, iso_dir, symlinks=True, dirs_exist_ok=True)
232
233 @classmethod
234 def do_prepare_partition(cls, part, source_params, creator, cr_workdir,
235 oe_builddir, bootimg_dir, kernel_dir,
236 rootfs_dir, native_sysroot):
237 """
238 Called to do the actual content population for a partition i.e. it
239 'prepares' the partition to be incorporated into the image.
240 In this case, prepare content for a bootable ISO image.
241 """
242
243 isodir = "%s/ISO" % cr_workdir
244
245 cls._install_payload(source_params, isodir)
246
247 if part.rootfs_dir is None:
248 if not 'ROOTFS_DIR' in rootfs_dir:
249 raise WicError("Couldn't find --rootfs-dir, exiting.")
250 rootfs_dir = rootfs_dir['ROOTFS_DIR']
251 else:
252 if part.rootfs_dir in rootfs_dir:
253 rootfs_dir = rootfs_dir[part.rootfs_dir]
254 elif part.rootfs_dir:
255 rootfs_dir = part.rootfs_dir
256 else:
257 raise WicError("Couldn't find --rootfs-dir=%s connection "
258 "or it is not a valid path, exiting." %
259 part.rootfs_dir)
260
261 if not os.path.isdir(rootfs_dir):
262 rootfs_dir = get_bitbake_var("IMAGE_ROOTFS")
263 if not os.path.isdir(rootfs_dir):
264 raise WicError("Couldn't find IMAGE_ROOTFS, exiting.")
265
266 part.rootfs_dir = rootfs_dir
267 deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
268 img_iso_dir = get_bitbake_var("ISODIR")
269
270 # Remove the temporary file created by part.prepare_rootfs()
271 if os.path.isfile(part.source_file):
272 os.remove(part.source_file)
273
274 # Support using a different initrd other than default
275 if source_params.get('initrd'):
276 initrd = source_params['initrd']
277 if not deploy_dir:
278 raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
279 cp_cmd = "cp %s/%s %s" % (deploy_dir, initrd, cr_workdir)
280 exec_cmd(cp_cmd)
281 else:
282 # Prepare initial ramdisk
283 initrd = "%s/initrd" % deploy_dir
284 if not os.path.isfile(initrd):
285 initrd = "%s/initrd" % img_iso_dir
286 if not os.path.isfile(initrd):
287 initrd = cls._build_initramfs_path(rootfs_dir, cr_workdir)
288
289 install_cmd = "install -m 0644 %s %s/initrd" % (initrd, isodir)
290 exec_cmd(install_cmd)
291
292 # Remove the temporary file created by _build_initramfs_path function
293 if os.path.isfile("%s/initrd.cpio.gz" % cr_workdir):
294 os.remove("%s/initrd.cpio.gz" % cr_workdir)
295
296 kernel = get_bitbake_var("KERNEL_IMAGETYPE")
297 if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1":
298 if get_bitbake_var("INITRAMFS_IMAGE"):
299 kernel = "%s-%s.bin" % \
300 (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
301
302 install_cmd = "install -m 0644 %s/%s %s/%s" % \
303 (kernel_dir, kernel, isodir, kernel)
304 exec_cmd(install_cmd)
305
306 #Create bootloader for efi boot
307 try:
308 target_dir = "%s/EFI/BOOT" % isodir
309 if os.path.exists(target_dir):
310 shutil.rmtree(target_dir)
311
312 os.makedirs(target_dir)
313
314 if source_params['loader'] == 'grub-efi':
315 # Builds bootx64.efi/bootia32.efi if ISODIR didn't exist or
316 # didn't contains it
317 target_arch = get_bitbake_var("TARGET_SYS")
318 if not target_arch:
319 raise WicError("Coludn't find target architecture")
320
321 if re.match("x86_64", target_arch):
322 grub_src_image = "grub-efi-bootx64.efi"
323 grub_dest_image = "bootx64.efi"
324 elif re.match('i.86', target_arch):
325 grub_src_image = "grub-efi-bootia32.efi"
326 grub_dest_image = "bootia32.efi"
327 else:
328 raise WicError("grub-efi is incompatible with target %s" %
329 target_arch)
330
331 grub_target = os.path.join(target_dir, grub_dest_image)
332 if not os.path.isfile(grub_target):
333 grub_src = os.path.join(deploy_dir, grub_src_image)
334 if not os.path.exists(grub_src):
335 raise WicError("Grub loader %s is not found in %s. "
336 "Please build grub-efi first" % (grub_src_image, deploy_dir))
337 shutil.copy(grub_src, grub_target)
338
339 if not os.path.isfile(os.path.join(target_dir, "boot.cfg")):
340 cls.do_configure_grubefi(part, creator, target_dir)
341
342 else:
343 raise WicError("unrecognized bootimg-efi loader: %s" %
344 source_params['loader'])
345 except KeyError:
346 raise WicError("bootimg-efi requires a loader, none specified")
347
348 # Create efi.img that contains bootloader files for EFI booting
349 # if ISODIR didn't exist or didn't contains it
350 if os.path.isfile("%s/efi.img" % img_iso_dir):
351 install_cmd = "install -m 0644 %s/efi.img %s/efi.img" % \
352 (img_iso_dir, isodir)
353 exec_cmd(install_cmd)
354 else:
355 # Default to 100 blocks of extra space for file system overhead
356 esp_extra_blocks = int(source_params.get('esp_extra_blocks', '100'))
357
358 du_cmd = "du -bks %s/EFI" % isodir
359 out = exec_cmd(du_cmd)
360 blocks = int(out.split()[0])
361 blocks += esp_extra_blocks
362 logger.debug("Added 100 extra blocks to %s to get to %d "
363 "total blocks", part.mountpoint, blocks)
364
365 # dosfs image for EFI boot
366 bootimg = "%s/efi.img" % isodir
367
368 esp_label = source_params.get('esp_label', 'EFIimg')
369
370 dosfs_cmd = 'mkfs.vfat -n \'%s\' -S 512 -C %s %d' \
371 % (esp_label, bootimg, blocks)
372 exec_native_cmd(dosfs_cmd, native_sysroot)
373
374 mmd_cmd = "mmd -i %s ::/EFI" % bootimg
375 exec_native_cmd(mmd_cmd, native_sysroot)
376
377 mcopy_cmd = "mcopy -i %s -s %s/EFI/* ::/EFI/" \
378 % (bootimg, isodir)
379 exec_native_cmd(mcopy_cmd, native_sysroot)
380
381 chmod_cmd = "chmod 644 %s" % bootimg
382 exec_cmd(chmod_cmd)
383
384 # Prepare files for legacy boot
385 syslinux_dir = get_bitbake_var("STAGING_DATADIR")
386 if not syslinux_dir:
387 raise WicError("Couldn't find STAGING_DATADIR, exiting.")
388
389 if os.path.exists("%s/isolinux" % isodir):
390 shutil.rmtree("%s/isolinux" % isodir)
391
392 install_cmd = "install -d %s/isolinux" % isodir
393 exec_cmd(install_cmd)
394
395 cls.do_configure_syslinux(creator, cr_workdir)
396
397 install_cmd = "install -m 444 %s/syslinux/ldlinux.sys " % syslinux_dir
398 install_cmd += "%s/isolinux/ldlinux.sys" % isodir
399 exec_cmd(install_cmd)
400
401 install_cmd = "install -m 444 %s/syslinux/isohdpfx.bin " % syslinux_dir
402 install_cmd += "%s/isolinux/isohdpfx.bin" % isodir
403 exec_cmd(install_cmd)
404
405 install_cmd = "install -m 644 %s/syslinux/isolinux.bin " % syslinux_dir
406 install_cmd += "%s/isolinux/isolinux.bin" % isodir
407 exec_cmd(install_cmd)
408
409 install_cmd = "install -m 644 %s/syslinux/ldlinux.c32 " % syslinux_dir
410 install_cmd += "%s/isolinux/ldlinux.c32" % isodir
411 exec_cmd(install_cmd)
412
413 #create ISO image
414 iso_img = "%s/tempiso_img.iso" % cr_workdir
415 iso_bootimg = "isolinux/isolinux.bin"
416 iso_bootcat = "isolinux/boot.cat"
417 efi_img = "efi.img"
418
419 mkisofs_cmd = "mkisofs -V %s " % part.label
420 mkisofs_cmd += "-o %s -U " % iso_img
421 mkisofs_cmd += "-J -joliet-long -r -iso-level 2 -b %s " % iso_bootimg
422 mkisofs_cmd += "-c %s -no-emul-boot -boot-load-size 4 " % iso_bootcat
423 mkisofs_cmd += "-boot-info-table -eltorito-alt-boot "
424 mkisofs_cmd += "-eltorito-platform 0xEF -eltorito-boot %s " % efi_img
425 mkisofs_cmd += "-no-emul-boot %s " % isodir
426
427 logger.debug("running command: %s", mkisofs_cmd)
428 exec_native_cmd(mkisofs_cmd, native_sysroot)
429
430 shutil.rmtree(isodir)
431
432 du_cmd = "du -Lbks %s" % iso_img
433 out = exec_cmd(du_cmd)
434 isoimg_size = int(out.split()[0])
435
436 part.size = isoimg_size
437 part.source_file = iso_img
438
439 @classmethod
440 def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir,
441 bootimg_dir, kernel_dir, native_sysroot):
442 """
443 Called after all partitions have been prepared and assembled into a
444 disk image. In this case, we insert/modify the MBR using isohybrid
445 utility for booting via BIOS from disk storage devices.
446 """
447
448 iso_img = "%s.p1" % disk.path
449 full_path = creator._full_path(workdir, disk_name, "direct")
450 full_path_iso = creator._full_path(workdir, disk_name, "iso")
451
452 isohybrid_cmd = "isohybrid -u %s" % iso_img
453 logger.debug("running command: %s", isohybrid_cmd)
454 exec_native_cmd(isohybrid_cmd, native_sysroot)
455
456 # Replace the image created by direct plugin with the one created by
457 # mkisofs command. This is necessary because the iso image created by
458 # mkisofs has a very specific MBR is system area of the ISO image, and
459 # direct plugin adds and configures an another MBR.
460 logger.debug("Replaceing the image created by direct plugin\n")
461 os.remove(disk.path)
462 shutil.copy2(iso_img, full_path_iso)
463 shutil.copy2(full_path_iso, full_path)
diff --git a/meta-xilinx-core/scripts/lib/wic/plugins/source/rawcopy.py b/meta-xilinx-core/scripts/lib/wic/plugins/source/rawcopy.py
new file mode 100644
index 00000000..21903c2f
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/plugins/source/rawcopy.py
@@ -0,0 +1,115 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import logging
8import os
9import signal
10import subprocess
11
12from wic import WicError
13from wic.pluginbase import SourcePlugin
14from wic.misc import exec_cmd, get_bitbake_var
15from wic.filemap import sparse_copy
16
17logger = logging.getLogger('wic')
18
19class RawCopyPlugin(SourcePlugin):
20 """
21 Populate partition content from raw image file.
22 """
23
24 name = 'rawcopy'
25
26 @staticmethod
27 def do_image_label(fstype, dst, label):
28 # don't create label when fstype is none
29 if fstype == 'none':
30 return
31
32 if fstype.startswith('ext'):
33 cmd = 'tune2fs -L %s %s' % (label, dst)
34 elif fstype in ('msdos', 'vfat'):
35 cmd = 'dosfslabel %s %s' % (dst, label)
36 elif fstype == 'btrfs':
37 cmd = 'btrfs filesystem label %s %s' % (dst, label)
38 elif fstype == 'swap':
39 cmd = 'mkswap -L %s %s' % (label, dst)
40 elif fstype in ('squashfs', 'erofs'):
41 raise WicError("It's not possible to update a %s "
42 "filesystem label '%s'" % (fstype, label))
43 else:
44 raise WicError("Cannot update filesystem label: "
45 "Unknown fstype: '%s'" % (fstype))
46
47 exec_cmd(cmd)
48
49 @staticmethod
50 def do_image_uncompression(src, dst, workdir):
51 def subprocess_setup():
52 # Python installs a SIGPIPE handler by default. This is usually not what
53 # non-Python subprocesses expect.
54 # SIGPIPE errors are known issues with gzip/bash
55 signal.signal(signal.SIGPIPE, signal.SIG_DFL)
56
57 extension = os.path.splitext(src)[1]
58 decompressor = {
59 ".bz2": "bzip2",
60 ".gz": "gzip",
61 ".xz": "xz",
62 ".zst": "zstd -f",
63 }.get(extension)
64 if not decompressor:
65 raise WicError("Not supported compressor filename extension: %s" % extension)
66 cmd = "%s -dc %s > %s" % (decompressor, src, dst)
67 subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=workdir)
68
69 @classmethod
70 def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
71 oe_builddir, bootimg_dir, kernel_dir,
72 rootfs_dir, native_sysroot):
73 """
74 Called to do the actual content population for a partition i.e. it
75 'prepares' the partition to be incorporated into the image.
76 """
77 if not kernel_dir:
78 kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
79 if not kernel_dir:
80 raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
81
82 logger.debug('Kernel dir: %s', kernel_dir)
83
84 if 'file' not in source_params:
85 raise WicError("No file specified")
86
87 if 'unpack' in source_params:
88 img = os.path.join(kernel_dir, source_params['file'])
89 src = os.path.join(cr_workdir, os.path.splitext(source_params['file'])[0])
90 RawCopyPlugin.do_image_uncompression(img, src, cr_workdir)
91 else:
92 src = os.path.join(kernel_dir, source_params['file'])
93
94 dst = os.path.join(cr_workdir, "%s.%s" % (os.path.basename(source_params['file']), part.lineno))
95
96 if not os.path.exists(os.path.dirname(dst)):
97 os.makedirs(os.path.dirname(dst))
98
99 if 'skip' in source_params:
100 sparse_copy(src, dst, skip=int(source_params['skip']))
101 else:
102 sparse_copy(src, dst)
103
104 # get the size in the right units for kickstart (kB)
105 du_cmd = "du -Lbks %s" % dst
106 out = exec_cmd(du_cmd)
107 filesize = int(out.split()[0])
108
109 if filesize > part.size:
110 part.size = filesize
111
112 if part.label:
113 RawCopyPlugin.do_image_label(part.fstype, dst, part.label)
114
115 part.source_file = dst
diff --git a/meta-xilinx-core/scripts/lib/wic/plugins/source/rootfs.py b/meta-xilinx-core/scripts/lib/wic/plugins/source/rootfs.py
new file mode 100644
index 00000000..e29f3a4c
--- /dev/null
+++ b/meta-xilinx-core/scripts/lib/wic/plugins/source/rootfs.py
@@ -0,0 +1,236 @@
1#
2# Copyright (c) 2014, Intel Corporation.
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6# DESCRIPTION
7# This implements the 'rootfs' source plugin class for 'wic'
8#
9# AUTHORS
10# Tom Zanussi <tom.zanussi (at] linux.intel.com>
11# Joao Henrique Ferreira de Freitas <joaohf (at] gmail.com>
12#
13
14import logging
15import os
16import shutil
17import sys
18
19from oe.path import copyhardlinktree
20from pathlib import Path
21
22from wic import WicError
23from wic.pluginbase import SourcePlugin
24from wic.misc import get_bitbake_var, exec_native_cmd
25
26logger = logging.getLogger('wic')
27
28class RootfsPlugin(SourcePlugin):
29 """
30 Populate partition content from a rootfs directory.
31 """
32
33 name = 'rootfs'
34
35 @staticmethod
36 def __validate_path(cmd, rootfs_dir, path):
37 if os.path.isabs(path):
38 logger.error("%s: Must be relative: %s" % (cmd, path))
39 sys.exit(1)
40
41 # Disallow climbing outside of parent directory using '..',
42 # because doing so could be quite disastrous (we will delete the
43 # directory, or modify a directory outside OpenEmbedded).
44 full_path = os.path.realpath(os.path.join(rootfs_dir, path))
45 if not full_path.startswith(os.path.realpath(rootfs_dir)):
46 logger.error("%s: Must point inside the rootfs:" % (cmd, path))
47 sys.exit(1)
48
49 return full_path
50
51 @staticmethod
52 def __get_rootfs_dir(rootfs_dir):
53 if rootfs_dir and os.path.isdir(rootfs_dir):
54 return os.path.realpath(rootfs_dir)
55
56 image_rootfs_dir = get_bitbake_var("IMAGE_ROOTFS", rootfs_dir)
57 if not os.path.isdir(image_rootfs_dir):
58 raise WicError("No valid artifact IMAGE_ROOTFS from image "
59 "named %s has been found at %s, exiting." %
60 (rootfs_dir, image_rootfs_dir))
61
62 return os.path.realpath(image_rootfs_dir)
63
64 @staticmethod
65 def __get_pseudo(native_sysroot, rootfs, pseudo_dir):
66 pseudo = "export PSEUDO_PREFIX=%s/usr;" % native_sysroot
67 pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % pseudo_dir
68 pseudo += "export PSEUDO_PASSWD=%s;" % rootfs
69 pseudo += "export PSEUDO_NOSYMLINKEXP=1;"
70 pseudo += "%s " % get_bitbake_var("FAKEROOTCMD")
71 return pseudo
72
73 @classmethod
74 def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
75 oe_builddir, bootimg_dir, kernel_dir,
76 krootfs_dir, native_sysroot):
77 """
78 Called to do the actual content population for a partition i.e. it
79 'prepares' the partition to be incorporated into the image.
80 In this case, prepare content for legacy bios boot partition.
81 """
82 if part.rootfs_dir is None:
83 if not 'ROOTFS_DIR' in krootfs_dir:
84 raise WicError("Couldn't find --rootfs-dir, exiting")
85
86 rootfs_dir = krootfs_dir['ROOTFS_DIR']
87 else:
88 if part.rootfs_dir in krootfs_dir:
89 rootfs_dir = krootfs_dir[part.rootfs_dir]
90 elif part.rootfs_dir:
91 rootfs_dir = part.rootfs_dir
92 else:
93 raise WicError("Couldn't find --rootfs-dir=%s connection or "
94 "it is not a valid path, exiting" % part.rootfs_dir)
95
96 part.rootfs_dir = cls.__get_rootfs_dir(rootfs_dir)
97 part.has_fstab = os.path.exists(os.path.join(part.rootfs_dir, "etc/fstab"))
98 pseudo_dir = os.path.join(part.rootfs_dir, "../pseudo")
99 if not os.path.lexists(pseudo_dir):
100 pseudo_dir = os.path.join(cls.__get_rootfs_dir(None), '../pseudo')
101
102 if not os.path.lexists(pseudo_dir):
103 logger.warn("%s folder does not exist. "
104 "Usernames and permissions will be invalid " % pseudo_dir)
105 pseudo_dir = None
106
107 new_rootfs = None
108 new_pseudo = None
109 # Handle excluded paths.
110 if part.exclude_path or part.include_path or part.change_directory or part.update_fstab_in_rootfs:
111 # We need a new rootfs directory we can safely modify without
112 # interfering with other tasks. Copy to workdir.
113 new_rootfs = os.path.realpath(os.path.join(cr_workdir, "rootfs%d" % part.lineno))
114
115 if os.path.lexists(new_rootfs):
116 shutil.rmtree(os.path.join(new_rootfs))
117
118 if part.change_directory:
119 cd = part.change_directory
120 if cd[-1] == '/':
121 cd = cd[:-1]
122 orig_dir = cls.__validate_path("--change-directory", part.rootfs_dir, cd)
123 else:
124 orig_dir = part.rootfs_dir
125 copyhardlinktree(orig_dir, new_rootfs)
126
127 # Convert the pseudo directory to its new location
128 if (pseudo_dir):
129 new_pseudo = os.path.realpath(
130 os.path.join(cr_workdir, "pseudo%d" % part.lineno))
131 if os.path.lexists(new_pseudo):
132 shutil.rmtree(new_pseudo)
133 os.mkdir(new_pseudo)
134 shutil.copy(os.path.join(pseudo_dir, "files.db"),
135 os.path.join(new_pseudo, "files.db"))
136
137 pseudo_cmd = "%s -B -m %s -M %s" % (cls.__get_pseudo(native_sysroot,
138 new_rootfs,
139 new_pseudo),
140 orig_dir, new_rootfs)
141 exec_native_cmd(pseudo_cmd, native_sysroot)
142
143 for in_path in part.include_path or []:
144 #parse arguments
145 include_path = in_path[0]
146 if len(in_path) > 2:
147 logger.error("'Invalid number of arguments for include-path")
148 sys.exit(1)
149 if len(in_path) == 2:
150 path = in_path[1]
151 else:
152 path = None
153
154 # Pack files to be included into a tar file.
155 # We need to create a tar file, because that way we can keep the
156 # permissions from the files even when they belong to different
157 # pseudo enviroments.
158 # If we simply copy files using copyhardlinktree/copytree... the
159 # copied files will belong to the user running wic.
160 tar_file = os.path.realpath(
161 os.path.join(cr_workdir, "include-path%d.tar" % part.lineno))
162 if os.path.isfile(include_path):
163 parent = os.path.dirname(os.path.realpath(include_path))
164 tar_cmd = "tar c --owner=root --group=root -f %s -C %s %s" % (
165 tar_file, parent, os.path.relpath(include_path, parent))
166 exec_native_cmd(tar_cmd, native_sysroot)
167 else:
168 if include_path in krootfs_dir:
169 include_path = krootfs_dir[include_path]
170 include_path = cls.__get_rootfs_dir(include_path)
171 include_pseudo = os.path.join(include_path, "../pseudo")
172 if os.path.lexists(include_pseudo):
173 pseudo = cls.__get_pseudo(native_sysroot, include_path,
174 include_pseudo)
175 tar_cmd = "tar cf %s -C %s ." % (tar_file, include_path)
176 else:
177 pseudo = None
178 tar_cmd = "tar c --owner=root --group=root -f %s -C %s ." % (
179 tar_file, include_path)
180 exec_native_cmd(tar_cmd, native_sysroot, pseudo)
181
182 #create destination
183 if path:
184 destination = cls.__validate_path("--include-path", new_rootfs, path)
185 Path(destination).mkdir(parents=True, exist_ok=True)
186 else:
187 destination = new_rootfs
188
189 #extract destination
190 untar_cmd = "tar xf %s -C %s" % (tar_file, destination)
191 if new_pseudo:
192 pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo)
193 else:
194 pseudo = None
195 exec_native_cmd(untar_cmd, native_sysroot, pseudo)
196 os.remove(tar_file)
197
198 for orig_path in part.exclude_path or []:
199 path = orig_path
200
201 full_path = cls.__validate_path("--exclude-path", new_rootfs, path)
202
203 if not os.path.lexists(full_path):
204 continue
205
206 if new_pseudo:
207 pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo)
208 else:
209 pseudo = None
210 if path.endswith(os.sep):
211 # Delete content only.
212 for entry in os.listdir(full_path):
213 full_entry = os.path.join(full_path, entry)
214 rm_cmd = "rm -rf %s" % (full_entry)
215 exec_native_cmd(rm_cmd, native_sysroot, pseudo)
216 else:
217 # Delete whole directory.
218 rm_cmd = "rm -rf %s" % (full_path)
219 exec_native_cmd(rm_cmd, native_sysroot, pseudo)
220
221 # Update part.has_fstab here as fstab may have been added or
222 # removed by the above modifications.
223 part.has_fstab = os.path.exists(os.path.join(new_rootfs, "etc/fstab"))
224 if part.update_fstab_in_rootfs and part.has_fstab and not part.no_fstab_update:
225 fstab_path = os.path.join(new_rootfs, "etc/fstab")
226 # Assume that fstab should always be owned by root with fixed permissions
227 install_cmd = "install -m 0644 -p %s %s" % (part.updated_fstab_path, fstab_path)
228 if new_pseudo:
229 pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo)
230 else:
231 pseudo = None
232 exec_native_cmd(install_cmd, native_sysroot, pseudo)
233
234 part.prepare_rootfs(cr_workdir, oe_builddir,
235 new_rootfs or part.rootfs_dir, native_sysroot,
236 pseudo_dir = new_pseudo or pseudo_dir)
diff --git a/meta-xilinx-core/scripts/wic b/meta-xilinx-core/scripts/wic
new file mode 100755
index 00000000..06e0b48d
--- /dev/null
+++ b/meta-xilinx-core/scripts/wic
@@ -0,0 +1,551 @@
1#!/usr/bin/env python3
2#
3# Copyright (c) 2013, Intel Corporation.
4#
5# SPDX-License-Identifier: GPL-2.0-only
6#
7# DESCRIPTION 'wic' is the OpenEmbedded Image Creator that users can
8# use to generate bootable images. Invoking it without any arguments
9# will display help screens for the 'wic' command and list the
10# available 'wic' subcommands. Invoking a subcommand without any
11# arguments will likewise display help screens for the specified
12# subcommand. Please use that interface for detailed help.
13#
14# AUTHORS
15# Tom Zanussi <tom.zanussi (at] linux.intel.com>
16#
17__version__ = "0.2.0"
18
19# Python Standard Library modules
20import os
21import sys
22import argparse
23import logging
24import subprocess
25import shutil
26
27from collections import namedtuple
28
29# External modules
30scripts_path = os.path.dirname(os.path.realpath(__file__))
31lib_path = scripts_path + '/lib'
32sys.path.insert(0, lib_path)
33import scriptpath
34scriptpath.add_oe_lib_path()
35
36# Check whether wic is running within eSDK environment
37sdkroot = scripts_path
38if os.environ.get('SDKTARGETSYSROOT'):
39 while sdkroot != '' and sdkroot != os.sep:
40 if os.path.exists(os.path.join(sdkroot, '.devtoolbase')):
41 # Set BUILDDIR for wic to work within eSDK
42 os.environ['BUILDDIR'] = sdkroot
43 # .devtoolbase only exists within eSDK
44 # If found, initialize bitbake path for eSDK environment and append to PATH
45 sdkroot = os.path.join(os.path.dirname(scripts_path), 'bitbake', 'bin')
46 os.environ['PATH'] += ":" + sdkroot
47 break
48 sdkroot = os.path.dirname(sdkroot)
49
50bitbake_exe = shutil.which('bitbake')
51if bitbake_exe:
52 bitbake_path = scriptpath.add_bitbake_lib_path()
53 import bb
54
55from wic import WicError
56from wic.misc import get_bitbake_var, BB_VARS
57from wic import engine
58from wic import help as hlp
59
60
61def wic_logger():
62 """Create and convfigure wic logger."""
63 logger = logging.getLogger('wic')
64 logger.setLevel(logging.INFO)
65
66 handler = logging.StreamHandler()
67
68 formatter = logging.Formatter('%(levelname)s: %(message)s')
69 handler.setFormatter(formatter)
70
71 logger.addHandler(handler)
72
73 return logger
74
75logger = wic_logger()
76
77def rootfs_dir_to_args(krootfs_dir):
78 """
79 Get a rootfs_dir dict and serialize to string
80 """
81 rootfs_dir = ''
82 for key, val in krootfs_dir.items():
83 rootfs_dir += ' '
84 rootfs_dir += '='.join([key, val])
85 return rootfs_dir.strip()
86
87
88class RootfsArgAction(argparse.Action):
89 def __init__(self, **kwargs):
90 super().__init__(**kwargs)
91
92 def __call__(self, parser, namespace, value, option_string=None):
93 if not "rootfs_dir" in vars(namespace) or \
94 not type(namespace.__dict__['rootfs_dir']) is dict:
95 namespace.__dict__['rootfs_dir'] = {}
96
97 if '=' in value:
98 (key, rootfs_dir) = value.split('=')
99 else:
100 key = 'ROOTFS_DIR'
101 rootfs_dir = value
102
103 namespace.__dict__['rootfs_dir'][key] = rootfs_dir
104
105
106def wic_create_subcommand(options, usage_str):
107 """
108 Command-line handling for image creation. The real work is done
109 by image.engine.wic_create()
110 """
111 if options.build_rootfs and not bitbake_exe:
112 raise WicError("Can't build rootfs as bitbake is not in the $PATH")
113
114 if not options.image_name:
115 missed = []
116 for val, opt in [(options.rootfs_dir, 'rootfs-dir'),
117 (options.bootimg_dir, 'bootimg-dir'),
118 (options.kernel_dir, 'kernel-dir'),
119 (options.native_sysroot, 'native-sysroot')]:
120 if not val:
121 missed.append(opt)
122 if missed:
123 raise WicError("The following build artifacts are not specified: %s" %
124 ", ".join(missed))
125
126 if options.image_name:
127 BB_VARS.default_image = options.image_name
128 else:
129 options.build_check = False
130
131 if options.vars_dir:
132 BB_VARS.vars_dir = options.vars_dir
133
134 if options.build_check and not engine.verify_build_env():
135 raise WicError("Couldn't verify build environment, exiting")
136
137 if options.debug:
138 logger.setLevel(logging.DEBUG)
139
140 if options.image_name:
141 if options.build_rootfs:
142 argv = ["bitbake", options.image_name]
143 if options.debug:
144 argv.append("--debug")
145
146 logger.info("Building rootfs...\n")
147 subprocess.check_call(argv)
148
149 rootfs_dir = get_bitbake_var("IMAGE_ROOTFS", options.image_name)
150 kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE", options.image_name)
151 bootimg_dir = get_bitbake_var("STAGING_DATADIR", options.image_name)
152
153 native_sysroot = options.native_sysroot
154 if options.vars_dir and not native_sysroot:
155 native_sysroot = get_bitbake_var("RECIPE_SYSROOT_NATIVE", options.image_name)
156 else:
157 if options.build_rootfs:
158 raise WicError("Image name is not specified, exiting. "
159 "(Use -e/--image-name to specify it)")
160 native_sysroot = options.native_sysroot
161
162 if options.kernel_dir:
163 kernel_dir = options.kernel_dir
164
165 if not options.vars_dir and (not native_sysroot or not os.path.isdir(native_sysroot)):
166 logger.info("Building wic-tools...\n")
167 subprocess.check_call(["bitbake", "wic-tools"])
168 native_sysroot = get_bitbake_var("RECIPE_SYSROOT_NATIVE", "wic-tools")
169
170 if not native_sysroot:
171 raise WicError("Unable to find the location of the native tools sysroot")
172
173 wks_file = options.wks_file
174
175 if not wks_file.endswith(".wks"):
176 wks_file = engine.find_canned_image(scripts_path, wks_file)
177 if not wks_file:
178 raise WicError("No image named %s found, exiting. (Use 'wic list images' "
179 "to list available images, or specify a fully-qualified OE "
180 "kickstart (.wks) filename)" % options.wks_file)
181
182 if not options.image_name:
183 rootfs_dir = ''
184 if 'ROOTFS_DIR' in options.rootfs_dir:
185 rootfs_dir = options.rootfs_dir['ROOTFS_DIR']
186 bootimg_dir = options.bootimg_dir
187 kernel_dir = options.kernel_dir
188 native_sysroot = options.native_sysroot
189 if rootfs_dir and not os.path.isdir(rootfs_dir):
190 raise WicError("--rootfs-dir (-r) not found, exiting")
191 if not os.path.isdir(bootimg_dir):
192 raise WicError("--bootimg-dir (-b) not found, exiting")
193 if not os.path.isdir(kernel_dir):
194 raise WicError("--kernel-dir (-k) not found, exiting")
195 if not os.path.isdir(native_sysroot):
196 raise WicError("--native-sysroot (-n) not found, exiting")
197 else:
198 not_found = not_found_dir = ""
199 if not os.path.isdir(rootfs_dir):
200 (not_found, not_found_dir) = ("rootfs-dir", rootfs_dir)
201 elif not os.path.isdir(kernel_dir):
202 (not_found, not_found_dir) = ("kernel-dir", kernel_dir)
203 elif not os.path.isdir(native_sysroot):
204 (not_found, not_found_dir) = ("native-sysroot", native_sysroot)
205 if not_found:
206 if not not_found_dir:
207 not_found_dir = "Completely missing artifact - wrong image (.wks) used?"
208 logger.info("Build artifacts not found, exiting.")
209 logger.info(" (Please check that the build artifacts for the machine")
210 logger.info(" selected in local.conf actually exist and that they")
211 logger.info(" are the correct artifacts for the image (.wks file)).\n")
212 raise WicError("The artifact that couldn't be found was %s:\n %s" % (not_found, not_found_dir))
213
214 krootfs_dir = options.rootfs_dir
215 if krootfs_dir is None:
216 krootfs_dir = {}
217 krootfs_dir['ROOTFS_DIR'] = rootfs_dir
218
219 rootfs_dir = rootfs_dir_to_args(krootfs_dir)
220
221 logger.info("Creating image(s)...\n")
222 engine.wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir,
223 native_sysroot, options)
224
225
226def wic_list_subcommand(args, usage_str):
227 """
228 Command-line handling for listing available images.
229 The real work is done by image.engine.wic_list()
230 """
231 if not engine.wic_list(args, scripts_path):
232 raise WicError("Bad list arguments, exiting")
233
234
235def wic_ls_subcommand(args, usage_str):
236 """
237 Command-line handling for list content of images.
238 The real work is done by engine.wic_ls()
239 """
240 engine.wic_ls(args, args.native_sysroot)
241
242def wic_cp_subcommand(args, usage_str):
243 """
244 Command-line handling for copying files/dirs to images.
245 The real work is done by engine.wic_cp()
246 """
247 engine.wic_cp(args, args.native_sysroot)
248
249def wic_rm_subcommand(args, usage_str):
250 """
251 Command-line handling for removing files/dirs from images.
252 The real work is done by engine.wic_rm()
253 """
254 engine.wic_rm(args, args.native_sysroot)
255
256def wic_write_subcommand(args, usage_str):
257 """
258 Command-line handling for writing images.
259 The real work is done by engine.wic_write()
260 """
261 engine.wic_write(args, args.native_sysroot)
262
263def wic_help_subcommand(args, usage_str):
264 """
265 Command-line handling for help subcommand to keep the current
266 structure of the function definitions.
267 """
268 pass
269
270
271def wic_help_topic_subcommand(usage_str, help_str):
272 """
273 Display function for help 'sub-subcommands'.
274 """
275 print(help_str)
276 return
277
278
279wic_help_topic_usage = """
280"""
281
282helptopics = {
283 "plugins": [wic_help_topic_subcommand,
284 wic_help_topic_usage,
285 hlp.wic_plugins_help],
286 "overview": [wic_help_topic_subcommand,
287 wic_help_topic_usage,
288 hlp.wic_overview_help],
289 "kickstart": [wic_help_topic_subcommand,
290 wic_help_topic_usage,
291 hlp.wic_kickstart_help],
292 "create": [wic_help_topic_subcommand,
293 wic_help_topic_usage,
294 hlp.wic_create_help],
295 "ls": [wic_help_topic_subcommand,
296 wic_help_topic_usage,
297 hlp.wic_ls_help],
298 "cp": [wic_help_topic_subcommand,
299 wic_help_topic_usage,
300 hlp.wic_cp_help],
301 "rm": [wic_help_topic_subcommand,
302 wic_help_topic_usage,
303 hlp.wic_rm_help],
304 "write": [wic_help_topic_subcommand,
305 wic_help_topic_usage,
306 hlp.wic_write_help],
307 "list": [wic_help_topic_subcommand,
308 wic_help_topic_usage,
309 hlp.wic_list_help]
310}
311
312
313def wic_init_parser_create(subparser):
314 subparser.add_argument("wks_file")
315
316 subparser.add_argument("-o", "--outdir", dest="outdir", default='.',
317 help="name of directory to create image in")
318 subparser.add_argument("-w", "--workdir",
319 help="temporary workdir to use for intermediate files")
320 subparser.add_argument("-e", "--image-name", dest="image_name",
321 help="name of the image to use the artifacts from "
322 "e.g. core-image-sato")
323 subparser.add_argument("-r", "--rootfs-dir", action=RootfsArgAction,
324 help="path to the /rootfs dir to use as the "
325 ".wks rootfs source")
326 subparser.add_argument("-b", "--bootimg-dir", dest="bootimg_dir",
327 help="path to the dir containing the boot artifacts "
328 "(e.g. /EFI or /syslinux dirs) to use as the "
329 ".wks bootimg source")
330 subparser.add_argument("-k", "--kernel-dir", dest="kernel_dir",
331 help="path to the dir containing the kernel to use "
332 "in the .wks bootimg")
333 subparser.add_argument("-n", "--native-sysroot", dest="native_sysroot",
334 help="path to the native sysroot containing the tools "
335 "to use to build the image")
336 subparser.add_argument("-s", "--skip-build-check", dest="build_check",
337 action="store_false", default=True, help="skip the build check")
338 subparser.add_argument("-f", "--build-rootfs", action="store_true", help="build rootfs")
339 subparser.add_argument("-c", "--compress-with", choices=("gzip", "bzip2", "xz"),
340 dest='compressor',
341 help="compress image with specified compressor")
342 subparser.add_argument("-m", "--bmap", action="store_true", help="generate .bmap")
343 subparser.add_argument("--no-fstab-update" ,action="store_true",
344 help="Do not change fstab file.")
345 subparser.add_argument("-v", "--vars", dest='vars_dir',
346 help="directory with <image>.env files that store "
347 "bitbake variables")
348 subparser.add_argument("-D", "--debug", dest="debug", action="store_true",
349 default=False, help="output debug information")
350 subparser.add_argument("-i", "--imager", dest="imager",
351 default="direct", help="the wic imager plugin")
352 subparser.add_argument("--extra-space", type=int, dest="extra_space",
353 default=0, help="additional free disk space to add to the image")
354 return
355
356
357def wic_init_parser_list(subparser):
358 subparser.add_argument("list_type",
359 help="can be 'images' or 'source-plugins' "
360 "to obtain a list. "
361 "If value is a valid .wks image file")
362 subparser.add_argument("help_for", default=[], nargs='*',
363 help="If 'list_type' is a valid .wks image file "
364 "this value can be 'help' to show the help information "
365 "defined inside the .wks file")
366 return
367
368def imgtype(arg):
369 """
370 Custom type for ArgumentParser
371 Converts path spec to named tuple: (image, partition, path)
372 """
373 image = arg
374 part = path = None
375 if ':' in image:
376 image, part = image.split(':')
377 if '/' in part:
378 part, path = part.split('/', 1)
379 if not path:
380 path = '/'
381
382 if not os.path.isfile(image):
383 err = "%s is not a regular file or symlink" % image
384 raise argparse.ArgumentTypeError(err)
385
386 return namedtuple('ImgType', 'image part path')(image, part, path)
387
388def wic_init_parser_ls(subparser):
389 subparser.add_argument("path", type=imgtype,
390 help="image spec: <image>[:<vfat partition>[<path>]]")
391 subparser.add_argument("-n", "--native-sysroot",
392 help="path to the native sysroot containing the tools")
393
394def imgpathtype(arg):
395 img = imgtype(arg)
396 if img.part is None:
397 raise argparse.ArgumentTypeError("partition number is not specified")
398 return img
399
400def wic_init_parser_cp(subparser):
401 subparser.add_argument("src",
402 help="image spec: <image>:<vfat partition>[<path>] or <file>")
403 subparser.add_argument("dest",
404 help="image spec: <image>:<vfat partition>[<path>] or <file>")
405 subparser.add_argument("-n", "--native-sysroot",
406 help="path to the native sysroot containing the tools")
407
408def wic_init_parser_rm(subparser):
409 subparser.add_argument("path", type=imgpathtype,
410 help="path: <image>:<vfat partition><path>")
411 subparser.add_argument("-n", "--native-sysroot",
412 help="path to the native sysroot containing the tools")
413 subparser.add_argument("-r", dest="recursive_delete", action="store_true", default=False,
414 help="remove directories and their contents recursively, "
415 " this only applies to ext* partition")
416
417def expandtype(rules):
418 """
419 Custom type for ArgumentParser
420 Converts expand rules to the dictionary {<partition>: size}
421 """
422 if rules == 'auto':
423 return {}
424 result = {}
425 for rule in rules.split(','):
426 try:
427 part, size = rule.split(':')
428 except ValueError:
429 raise argparse.ArgumentTypeError("Incorrect rule format: %s" % rule)
430
431 if not part.isdigit():
432 raise argparse.ArgumentTypeError("Rule '%s': partition number must be integer" % rule)
433
434 # validate size
435 multiplier = 1
436 for suffix, mult in [('K', 1024), ('M', 1024 * 1024), ('G', 1024 * 1024 * 1024)]:
437 if size.upper().endswith(suffix):
438 multiplier = mult
439 size = size[:-1]
440 break
441 if not size.isdigit():
442 raise argparse.ArgumentTypeError("Rule '%s': size must be integer" % rule)
443
444 result[int(part)] = int(size) * multiplier
445
446 return result
447
448def wic_init_parser_write(subparser):
449 subparser.add_argument("image",
450 help="path to the wic image")
451 subparser.add_argument("target",
452 help="target file or device")
453 subparser.add_argument("-e", "--expand", type=expandtype,
454 help="expand rules: auto or <partition>:<size>[,<partition>:<size>]")
455 subparser.add_argument("-n", "--native-sysroot",
456 help="path to the native sysroot containing the tools")
457
458def wic_init_parser_help(subparser):
459 helpparsers = subparser.add_subparsers(dest='help_topic', help=hlp.wic_usage)
460 for helptopic in helptopics:
461 helpparsers.add_parser(helptopic, help=helptopics[helptopic][2])
462 return
463
464
465subcommands = {
466 "create": [wic_create_subcommand,
467 hlp.wic_create_usage,
468 hlp.wic_create_help,
469 wic_init_parser_create],
470 "list": [wic_list_subcommand,
471 hlp.wic_list_usage,
472 hlp.wic_list_help,
473 wic_init_parser_list],
474 "ls": [wic_ls_subcommand,
475 hlp.wic_ls_usage,
476 hlp.wic_ls_help,
477 wic_init_parser_ls],
478 "cp": [wic_cp_subcommand,
479 hlp.wic_cp_usage,
480 hlp.wic_cp_help,
481 wic_init_parser_cp],
482 "rm": [wic_rm_subcommand,
483 hlp.wic_rm_usage,
484 hlp.wic_rm_help,
485 wic_init_parser_rm],
486 "write": [wic_write_subcommand,
487 hlp.wic_write_usage,
488 hlp.wic_write_help,
489 wic_init_parser_write],
490 "help": [wic_help_subcommand,
491 wic_help_topic_usage,
492 hlp.wic_help_help,
493 wic_init_parser_help]
494}
495
496
497def init_parser(parser):
498 parser.add_argument("--version", action="version",
499 version="%(prog)s {version}".format(version=__version__))
500 parser.add_argument("-D", "--debug", dest="debug", action="store_true",
501 default=False, help="output debug information")
502
503 subparsers = parser.add_subparsers(dest='command', help=hlp.wic_usage)
504 for subcmd in subcommands:
505 subparser = subparsers.add_parser(subcmd, help=subcommands[subcmd][2])
506 subcommands[subcmd][3](subparser)
507
508class WicArgumentParser(argparse.ArgumentParser):
509 def format_help(self):
510 return hlp.wic_help
511
512def main(argv):
513 parser = WicArgumentParser(
514 description="wic version %s" % __version__)
515
516 init_parser(parser)
517
518 args = parser.parse_args(argv)
519
520 if args.debug:
521 logger.setLevel(logging.DEBUG)
522
523 if "command" in vars(args):
524 if args.command == "help":
525 if args.help_topic is None:
526 parser.print_help()
527 elif args.help_topic in helptopics:
528 hlpt = helptopics[args.help_topic]
529 hlpt[0](hlpt[1], hlpt[2])
530 return 0
531
532 # validate wic cp src and dest parameter to identify which one of it is
533 # image and cast it into imgtype
534 if args.command == "cp":
535 if ":" in args.dest:
536 args.dest = imgtype(args.dest)
537 elif ":" in args.src:
538 args.src = imgtype(args.src)
539 else:
540 raise argparse.ArgumentTypeError("no image or partition number specified.")
541
542 return hlp.invoke_subcommand(args, parser, hlp.wic_help_usage, subcommands)
543
544
545if __name__ == "__main__":
546 try:
547 sys.exit(main(sys.argv[1:]))
548 except WicError as err:
549 print()
550 logger.error(err)
551 sys.exit(1)