diff options
Diffstat (limited to 'meta')
| -rw-r--r-- | meta/classes/image.bbclass | 227 | ||||
| -rw-r--r-- | meta/classes/rootfs-postcommands.bbclass | 13 | ||||
| -rw-r--r-- | meta/lib/oe/image.py | 412 |
3 files changed, 216 insertions, 436 deletions
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass index ffad5d28d2..5003dacc1c 100644 --- a/meta/classes/image.bbclass +++ b/meta/classes/image.bbclass | |||
| @@ -106,21 +106,6 @@ python () { | |||
| 106 | d.setVarFlag(var, 'func', '1') | 106 | d.setVarFlag(var, 'func', '1') |
| 107 | } | 107 | } |
| 108 | 108 | ||
| 109 | def fstype_variables(d): | ||
| 110 | import oe.image | ||
| 111 | |||
| 112 | image = oe.image.Image(d) | ||
| 113 | alltypes, fstype_groups, cimages = image._get_image_types() | ||
| 114 | fstype_vars = set() | ||
| 115 | for fstype_group in fstype_groups: | ||
| 116 | for fstype in fstype_group: | ||
| 117 | fstype_vars.add('IMAGE_CMD_' + fstype) | ||
| 118 | if fstype in cimages: | ||
| 119 | for ctype in cimages[fstype]: | ||
| 120 | fstype_vars.add('COMPRESS_CMD_' + ctype) | ||
| 121 | |||
| 122 | return sorted(fstype_vars) | ||
| 123 | |||
| 124 | def rootfs_variables(d): | 109 | def rootfs_variables(d): |
| 125 | from oe.rootfs import variable_depends | 110 | from oe.rootfs import variable_depends |
| 126 | variables = ['IMAGE_DEVICE_TABLES','BUILD_IMAGES_FROM_FEEDS','IMAGE_TYPES_MASKED','IMAGE_ROOTFS_ALIGNMENT','IMAGE_OVERHEAD_FACTOR','IMAGE_ROOTFS_SIZE','IMAGE_ROOTFS_EXTRA_SPACE', | 111 | variables = ['IMAGE_DEVICE_TABLES','BUILD_IMAGES_FROM_FEEDS','IMAGE_TYPES_MASKED','IMAGE_ROOTFS_ALIGNMENT','IMAGE_OVERHEAD_FACTOR','IMAGE_ROOTFS_SIZE','IMAGE_ROOTFS_EXTRA_SPACE', |
| @@ -129,7 +114,6 @@ def rootfs_variables(d): | |||
| 129 | 'MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS','PACKAGE_ARCHS', | 114 | 'MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS','PACKAGE_ARCHS', |
| 130 | 'PACKAGE_CLASSES','TARGET_VENDOR','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS', | 115 | 'PACKAGE_CLASSES','TARGET_VENDOR','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS', |
| 131 | 'COMPRESSIONTYPES', 'IMAGE_GEN_DEBUGFS'] | 116 | 'COMPRESSIONTYPES', 'IMAGE_GEN_DEBUGFS'] |
| 132 | variables.extend(fstype_variables(d)) | ||
| 133 | variables.extend(command_variables(d)) | 117 | variables.extend(command_variables(d)) |
| 134 | variables.extend(variable_depends(d)) | 118 | variables.extend(variable_depends(d)) |
| 135 | return " ".join(variables) | 119 | return " ".join(variables) |
| @@ -250,19 +234,13 @@ do_rootfs[umask] = "022" | |||
| 250 | addtask rootfs before do_build | 234 | addtask rootfs before do_build |
| 251 | 235 | ||
| 252 | fakeroot python do_image () { | 236 | fakeroot python do_image () { |
| 253 | from oe.image import create_image | ||
| 254 | from oe.image import Image | ||
| 255 | from oe.utils import execute_pre_post_process | 237 | from oe.utils import execute_pre_post_process |
| 256 | 238 | ||
| 257 | i = Image(d) | ||
| 258 | |||
| 259 | pre_process_cmds = d.getVar("IMAGE_PREPROCESS_COMMAND", True) | 239 | pre_process_cmds = d.getVar("IMAGE_PREPROCESS_COMMAND", True) |
| 260 | 240 | ||
| 261 | execute_pre_post_process(d, pre_process_cmds) | 241 | execute_pre_post_process(d, pre_process_cmds) |
| 262 | 242 | ||
| 263 | i._remove_old_symlinks() | 243 | write_wic_env(d) |
| 264 | |||
| 265 | i.create() | ||
| 266 | } | 244 | } |
| 267 | do_image[dirs] = "${TOPDIR}" | 245 | do_image[dirs] = "${TOPDIR}" |
| 268 | do_image[umask] = "022" | 246 | do_image[umask] = "022" |
| @@ -279,6 +257,209 @@ do_image_complete[dirs] = "${TOPDIR}" | |||
| 279 | do_image_complete[umask] = "022" | 257 | do_image_complete[umask] = "022" |
| 280 | addtask do_image_complete after do_image before do_build | 258 | addtask do_image_complete after do_image before do_build |
| 281 | 259 | ||
| 260 | # | ||
| 261 | # Write environment variables used by wic | ||
| 262 | # to tmp/sysroots/<machine>/imgdata/<image>.env | ||
| 263 | # | ||
| 264 | def write_wic_env(d): | ||
| 265 | wicvars = d.getVar('WICVARS', True) | ||
| 266 | if not wicvars: | ||
| 267 | return | ||
| 268 | |||
| 269 | stdir = d.getVar('STAGING_DIR_TARGET', True) | ||
| 270 | outdir = os.path.join(stdir, 'imgdata') | ||
| 271 | bb.utils.mkdirhier(outdir) | ||
| 272 | basename = d.getVar('IMAGE_BASENAME', True) | ||
| 273 | with open(os.path.join(outdir, basename) + '.env', 'w') as envf: | ||
| 274 | for var in wicvars.split(): | ||
| 275 | value = d.getVar(var, True) | ||
| 276 | if value: | ||
| 277 | envf.write('%s="%s"\n' % (var, value.strip())) | ||
| 278 | |||
| 279 | def setup_debugfs_variables(d): | ||
| 280 | d.appendVar('IMAGE_ROOTFS', '-dbg') | ||
| 281 | d.appendVar('IMAGE_LINK_NAME', '-dbg') | ||
| 282 | d.appendVar('IMAGE_NAME','-dbg') | ||
| 283 | debugfs_image_fstypes = d.getVar('IMAGE_FSTYPES_DEBUGFS', True) | ||
| 284 | if debugfs_image_fstypes: | ||
| 285 | d.setVar('IMAGE_FSTYPES', debugfs_image_fstypes) | ||
| 286 | |||
| 287 | python setup_debugfs () { | ||
| 288 | setup_debugfs_variables(d) | ||
| 289 | } | ||
| 290 | |||
| 291 | python () { | ||
| 292 | vardeps = set() | ||
| 293 | ctypes = d.getVar('COMPRESSIONTYPES', True).split() | ||
| 294 | old_overrides = d.getVar('OVERRIDES', 0) | ||
| 295 | |||
| 296 | def _image_base_type(type): | ||
| 297 | if type in ["vmdk", "vdi", "qcow2", "live", "iso", "hddimg"]: | ||
| 298 | type = "ext4" | ||
| 299 | basetype = type | ||
| 300 | for ctype in ctypes: | ||
| 301 | if type.endswith("." + ctype): | ||
| 302 | basetype = type[:-len("." + ctype)] | ||
| 303 | break | ||
| 304 | |||
| 305 | return basetype | ||
| 306 | |||
| 307 | basetypes = {} | ||
| 308 | alltypes = d.getVar('IMAGE_FSTYPES', True).split() | ||
| 309 | typedeps = {} | ||
| 310 | |||
| 311 | if d.getVar('IMAGE_GEN_DEBUGFS', True) == "1": | ||
| 312 | debugfs_fstypes = d.getVar('IMAGE_FSTYPES_DEBUGFS', True).split() | ||
| 313 | for t in debugfs_fstypes: | ||
| 314 | alltypes.append("debugfs_" + t) | ||
| 315 | |||
| 316 | def _add_type(t): | ||
| 317 | baset = _image_base_type(t) | ||
| 318 | if baset not in basetypes: | ||
| 319 | basetypes[baset]= [] | ||
| 320 | if t not in basetypes[baset]: | ||
| 321 | basetypes[baset].append(t) | ||
| 322 | debug = "" | ||
| 323 | if t.startswith("debugfs_"): | ||
| 324 | t = t[8:] | ||
| 325 | debug = "debugfs_" | ||
| 326 | deps = (d.getVar('IMAGE_TYPEDEP_' + t, True) or "").split() | ||
| 327 | vardeps.add('IMAGE_TYPEDEP_' + t) | ||
| 328 | if baset not in typedeps: | ||
| 329 | typedeps[baset] = set() | ||
| 330 | deps = [debug + dep for dep in deps] | ||
| 331 | for dep in deps: | ||
| 332 | if dep not in alltypes: | ||
| 333 | alltypes.append(dep) | ||
| 334 | _add_type(dep) | ||
| 335 | basedep = _image_base_type(dep) | ||
| 336 | typedeps[baset].add(basedep) | ||
| 337 | |||
| 338 | for t in alltypes[:]: | ||
| 339 | _add_type(t) | ||
| 340 | |||
| 341 | d.appendVarFlag('do_image', 'vardeps', ' '.join(vardeps)) | ||
| 342 | |||
| 343 | for t in basetypes: | ||
| 344 | vardeps = set() | ||
| 345 | cmds = [] | ||
| 346 | subimages = [] | ||
| 347 | realt = t | ||
| 348 | |||
| 349 | localdata = bb.data.createCopy(d) | ||
| 350 | debug = "" | ||
| 351 | if t.startswith("debugfs_"): | ||
| 352 | setup_debugfs_variables(localdata) | ||
| 353 | debug = "setup_debugfs " | ||
| 354 | realt = t[8:] | ||
| 355 | localdata.setVar('OVERRIDES', '%s:%s' % (realt, old_overrides)) | ||
| 356 | bb.data.update_data(localdata) | ||
| 357 | localdata.setVar('type', realt) | ||
| 358 | |||
| 359 | image_cmd = localdata.getVar("IMAGE_CMD", True) | ||
| 360 | vardeps.add('IMAGE_CMD_' + realt) | ||
| 361 | if image_cmd: | ||
| 362 | cmds.append("\t" + image_cmd) | ||
| 363 | else: | ||
| 364 | bb.fatal("No IMAGE_CMD defined for IMAGE_FSTYPES entry '%s' - possibly invalid type name or missing support class" % t) | ||
| 365 | cmds.append(localdata.expand("\tcd ${DEPLOY_DIR_IMAGE}")) | ||
| 366 | |||
| 367 | for bt in basetypes[t]: | ||
| 368 | for ctype in ctypes: | ||
| 369 | if bt.endswith("." + ctype): | ||
| 370 | cmds.append("\t" + localdata.getVar("COMPRESS_CMD_" + ctype, True)) | ||
| 371 | vardeps.add('COMPRESS_CMD_' + ctype) | ||
| 372 | subimages.append(realt + "." + ctype) | ||
| 373 | |||
| 374 | if realt not in alltypes: | ||
| 375 | cmds.append(localdata.expand("\trm ${IMAGE_NAME}.rootfs.${type}")) | ||
| 376 | else: | ||
| 377 | subimages.append(realt) | ||
| 378 | |||
| 379 | d.setVar('do_image_%s' % t, '\n'.join(cmds)) | ||
| 380 | d.setVarFlag('do_image_%s' % t, 'func', '1') | ||
| 381 | d.setVarFlag('do_image_%s' % t, 'fakeroot', '1') | ||
| 382 | d.setVarFlag('do_image_%s' % t, 'prefuncs', debug + 'set_image_size') | ||
| 383 | d.setVarFlag('do_image_%s' % t, 'postfuncs', 'create_symlinks') | ||
| 384 | d.setVarFlag('do_image_%s' % t, 'subimages', subimages) | ||
| 385 | d.appendVarFlag('do_image_%s' % t, 'vardeps', ' '.join(vardeps)) | ||
| 386 | |||
| 387 | after = 'do_image' | ||
| 388 | for dep in typedeps[t]: | ||
| 389 | after += ' do_image_%s' % dep | ||
| 390 | |||
| 391 | bb.debug(2, "Adding type %s before %s, after %s" % (t, 'do_image_complete', after)) | ||
| 392 | bb.build.addtask('do_image_%s' % t, 'do_image_complete', after, d) | ||
| 393 | } | ||
| 394 | |||
| 395 | # | ||
| 396 | # Compute the rootfs size | ||
| 397 | # | ||
| 398 | def get_rootfs_size(d): | ||
| 399 | import subprocess | ||
| 400 | |||
| 401 | rootfs_alignment = int(d.getVar('IMAGE_ROOTFS_ALIGNMENT', True)) | ||
| 402 | overhead_factor = float(d.getVar('IMAGE_OVERHEAD_FACTOR', True)) | ||
| 403 | rootfs_req_size = int(d.getVar('IMAGE_ROOTFS_SIZE', True)) | ||
| 404 | rootfs_extra_space = eval(d.getVar('IMAGE_ROOTFS_EXTRA_SPACE', True)) | ||
| 405 | rootfs_maxsize = d.getVar('IMAGE_ROOTFS_MAXSIZE', True) | ||
| 406 | |||
| 407 | output = subprocess.check_output(['du', '-ks', | ||
| 408 | d.getVar('IMAGE_ROOTFS', True)]) | ||
| 409 | size_kb = int(output.split()[0]) | ||
| 410 | base_size = size_kb * overhead_factor | ||
| 411 | base_size = (base_size, rootfs_req_size)[base_size < rootfs_req_size] + \ | ||
| 412 | rootfs_extra_space | ||
| 413 | |||
| 414 | if base_size != int(base_size): | ||
| 415 | base_size = int(base_size + 1) | ||
| 416 | else: | ||
| 417 | base_size = int(base_size) | ||
| 418 | |||
| 419 | base_size += rootfs_alignment - 1 | ||
| 420 | base_size -= base_size % rootfs_alignment | ||
| 421 | |||
| 422 | # Check the rootfs size against IMAGE_ROOTFS_MAXSIZE (if set) | ||
| 423 | if rootfs_maxsize: | ||
| 424 | rootfs_maxsize_int = int(rootfs_maxsize) | ||
| 425 | if base_size > rootfs_maxsize_int: | ||
| 426 | bb.fatal("The rootfs size %d(K) overrides the max size %d(K)" % \ | ||
| 427 | (base_size, rootfs_maxsize_int)) | ||
| 428 | return base_size | ||
| 429 | |||
| 430 | python set_image_size () { | ||
| 431 | rootfs_size = get_rootfs_size(d) | ||
| 432 | d.setVar('ROOTFS_SIZE', str(rootfs_size)) | ||
| 433 | d.setVarFlag('ROOTFS_SIZE', 'export', '1') | ||
| 434 | } | ||
| 435 | |||
| 436 | # | ||
| 437 | # Create symlinks to the newly created image | ||
| 438 | # | ||
| 439 | python create_symlinks() { | ||
| 440 | |||
| 441 | deploy_dir = d.getVar('DEPLOY_DIR_IMAGE', True) | ||
| 442 | img_name = d.getVar('IMAGE_NAME', True) | ||
| 443 | link_name = d.getVar('IMAGE_LINK_NAME', True) | ||
| 444 | manifest_name = d.getVar('IMAGE_MANIFEST', True) | ||
| 445 | taskname = d.getVar("BB_CURRENTTASK", True) | ||
| 446 | subimages = d.getVarFlag("do_" + taskname, 'subimages', False) | ||
| 447 | os.chdir(deploy_dir) | ||
| 448 | |||
| 449 | if not link_name: | ||
| 450 | return | ||
| 451 | for type in subimages: | ||
| 452 | if os.path.exists(img_name + ".rootfs." + type): | ||
| 453 | dst = deploy_dir + "/" + link_name + "." + type | ||
| 454 | src = img_name + ".rootfs." + type | ||
| 455 | bb.note("Creating symlink: %s -> %s" % (dst, src)) | ||
| 456 | if os.path.islink(dst): | ||
| 457 | if d.getVar('RM_OLD_IMAGE', True) == "1" and \ | ||
| 458 | os.path.exists(os.path.realpath(dst)): | ||
| 459 | os.remove(os.path.realpath(dst)) | ||
| 460 | os.remove(dst) | ||
| 461 | os.symlink(src, dst) | ||
| 462 | } | ||
| 282 | 463 | ||
| 283 | MULTILIBRE_ALLOW_REP =. "${base_bindir}|${base_sbindir}|${bindir}|${sbindir}|${libexecdir}|${sysconfdir}|${nonarch_base_libdir}/udev|/lib/modules/[^/]*/modules.*|" | 464 | MULTILIBRE_ALLOW_REP =. "${base_bindir}|${base_sbindir}|${bindir}|${sbindir}|${libexecdir}|${sysconfdir}|${nonarch_base_libdir}/udev|/lib/modules/[^/]*/modules.*|" |
| 284 | MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py" | 465 | MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py" |
diff --git a/meta/classes/rootfs-postcommands.bbclass b/meta/classes/rootfs-postcommands.bbclass index 7c440acf00..96d3051429 100644 --- a/meta/classes/rootfs-postcommands.bbclass +++ b/meta/classes/rootfs-postcommands.bbclass | |||
| @@ -207,9 +207,20 @@ insert_feed_uris () { | |||
| 207 | 207 | ||
| 208 | python write_image_manifest () { | 208 | python write_image_manifest () { |
| 209 | from oe.rootfs import image_list_installed_packages | 209 | from oe.rootfs import image_list_installed_packages |
| 210 | with open(d.getVar('IMAGE_MANIFEST', True), 'w+') as image_manifest: | 210 | |
| 211 | deploy_dir = d.getVar('DEPLOY_DIR_IMAGE', True) | ||
| 212 | link_name = d.getVar('IMAGE_LINK_NAME', True) | ||
| 213 | manifest_name = d.getVar('IMAGE_MANIFEST', True) | ||
| 214 | |||
| 215 | with open(manifest_name, 'w+') as image_manifest: | ||
| 211 | image_manifest.write(image_list_installed_packages(d, 'ver')) | 216 | image_manifest.write(image_list_installed_packages(d, 'ver')) |
| 212 | image_manifest.write("\n") | 217 | image_manifest.write("\n") |
| 218 | |||
| 219 | if manifest_name is not None and os.path.exists(manifest_name): | ||
| 220 | manifest_link = deploy_dir + "/" + link_name + ".manifest" | ||
| 221 | if os.path.exists(manifest_link): | ||
| 222 | os.remove(manifest_link) | ||
| 223 | os.symlink(os.path.basename(manifest_name), manifest_link) | ||
| 213 | } | 224 | } |
| 214 | 225 | ||
| 215 | # Can be use to create /etc/timestamp during image construction to give a reasonably | 226 | # Can be use to create /etc/timestamp during image construction to give a reasonably |
diff --git a/meta/lib/oe/image.py b/meta/lib/oe/image.py deleted file mode 100644 index b2b002b190..0000000000 --- a/meta/lib/oe/image.py +++ /dev/null | |||
| @@ -1,412 +0,0 @@ | |||
| 1 | from oe.utils import execute_pre_post_process | ||
| 2 | import os | ||
| 3 | import subprocess | ||
| 4 | import multiprocessing | ||
| 5 | |||
| 6 | |||
| 7 | def generate_image(arg): | ||
| 8 | (type, subimages, create_img_cmd, sprefix) = arg | ||
| 9 | |||
| 10 | bb.note("Running image creation script for %s: %s ..." % | ||
| 11 | (type, create_img_cmd)) | ||
| 12 | |||
| 13 | try: | ||
| 14 | output = subprocess.check_output(create_img_cmd, | ||
| 15 | stderr=subprocess.STDOUT) | ||
| 16 | except subprocess.CalledProcessError as e: | ||
| 17 | return("Error: The image creation script '%s' returned %d:\n%s" % | ||
| 18 | (e.cmd, e.returncode, e.output)) | ||
| 19 | |||
| 20 | bb.note("Script output:\n%s" % output) | ||
| 21 | |||
| 22 | return None | ||
| 23 | |||
| 24 | |||
| 25 | """ | ||
| 26 | This class will help compute IMAGE_FSTYPE dependencies and group them in batches | ||
| 27 | that can be executed in parallel. | ||
| 28 | |||
| 29 | The next example is for illustration purposes, highly unlikely to happen in real life. | ||
| 30 | It's just one of the test cases I used to test the algorithm: | ||
| 31 | |||
| 32 | For: | ||
| 33 | IMAGE_FSTYPES = "i1 i2 i3 i4 i5" | ||
| 34 | IMAGE_TYPEDEP_i4 = "i2" | ||
| 35 | IMAGE_TYPEDEP_i5 = "i6 i4" | ||
| 36 | IMAGE_TYPEDEP_i6 = "i7" | ||
| 37 | IMAGE_TYPEDEP_i7 = "i2" | ||
| 38 | |||
| 39 | We get the following list of batches that can be executed in parallel, having the | ||
| 40 | dependencies satisfied: | ||
| 41 | |||
| 42 | [['i1', 'i3', 'i2'], ['i4', 'i7'], ['i6'], ['i5']] | ||
| 43 | """ | ||
| 44 | class ImageDepGraph(object): | ||
| 45 | def __init__(self, d): | ||
| 46 | self.d = d | ||
| 47 | self.graph = dict() | ||
| 48 | self.deps_array = dict() | ||
| 49 | |||
| 50 | def _construct_dep_graph(self, image_fstypes): | ||
| 51 | graph = dict() | ||
| 52 | |||
| 53 | def add_node(node): | ||
| 54 | base_type = self._image_base_type(node) | ||
| 55 | deps = (self.d.getVar('IMAGE_TYPEDEP_' + node, True) or "") | ||
| 56 | base_deps = (self.d.getVar('IMAGE_TYPEDEP_' + base_type, True) or "") | ||
| 57 | |||
| 58 | graph[node] = "" | ||
| 59 | for dep in deps.split() + base_deps.split(): | ||
| 60 | if not dep in graph[node]: | ||
| 61 | if graph[node] != "": | ||
| 62 | graph[node] += " " | ||
| 63 | graph[node] += dep | ||
| 64 | |||
| 65 | if not dep in graph: | ||
| 66 | add_node(dep) | ||
| 67 | |||
| 68 | for fstype in image_fstypes: | ||
| 69 | add_node(fstype) | ||
| 70 | |||
| 71 | return graph | ||
| 72 | |||
| 73 | def _clean_graph(self): | ||
| 74 | # Live and VMDK/VDI images will be processed via inheriting | ||
| 75 | # bbclass and does not get processed here. Remove them from the fstypes | ||
| 76 | # graph. Their dependencies are already added, so no worries here. | ||
| 77 | remove_list = (self.d.getVar('IMAGE_TYPES_MASKED', True) or "").split() | ||
| 78 | |||
| 79 | for item in remove_list: | ||
| 80 | self.graph.pop(item, None) | ||
| 81 | |||
| 82 | def _image_base_type(self, type): | ||
| 83 | ctypes = self.d.getVar('COMPRESSIONTYPES', True).split() | ||
| 84 | if type in ["vmdk", "vdi", "qcow2", "live", "iso", "hddimg"]: | ||
| 85 | type = "ext4" | ||
| 86 | basetype = type | ||
| 87 | for ctype in ctypes: | ||
| 88 | if type.endswith("." + ctype): | ||
| 89 | basetype = type[:-len("." + ctype)] | ||
| 90 | break | ||
| 91 | |||
| 92 | return basetype | ||
| 93 | |||
| 94 | def _compute_dependencies(self): | ||
| 95 | """ | ||
| 96 | returns dict object of nodes with [no_of_depends_on, no_of_depended_by] | ||
| 97 | for each node | ||
| 98 | """ | ||
| 99 | deps_array = dict() | ||
| 100 | for node in self.graph: | ||
| 101 | deps_array[node] = [0, 0] | ||
| 102 | |||
| 103 | for node in self.graph: | ||
| 104 | deps = self.graph[node].split() | ||
| 105 | deps_array[node][0] += len(deps) | ||
| 106 | for dep in deps: | ||
| 107 | deps_array[dep][1] += 1 | ||
| 108 | |||
| 109 | return deps_array | ||
| 110 | |||
| 111 | def _sort_graph(self): | ||
| 112 | sorted_list = [] | ||
| 113 | group = [] | ||
| 114 | for node in self.graph: | ||
| 115 | if node not in self.deps_array: | ||
| 116 | continue | ||
| 117 | |||
| 118 | depends_on = self.deps_array[node][0] | ||
| 119 | |||
| 120 | if depends_on == 0: | ||
| 121 | group.append(node) | ||
| 122 | |||
| 123 | if len(group) == 0 and len(self.deps_array) != 0: | ||
| 124 | bb.fatal("possible fstype circular dependency...") | ||
| 125 | |||
| 126 | sorted_list.append(group) | ||
| 127 | |||
| 128 | # remove added nodes from deps_array | ||
| 129 | for item in group: | ||
| 130 | for node in self.graph: | ||
| 131 | if item in self.graph[node].split(): | ||
| 132 | self.deps_array[node][0] -= 1 | ||
| 133 | |||
| 134 | self.deps_array.pop(item, None) | ||
| 135 | |||
| 136 | if len(self.deps_array): | ||
| 137 | # recursive call, to find the next group | ||
| 138 | sorted_list += self._sort_graph() | ||
| 139 | |||
| 140 | return sorted_list | ||
| 141 | |||
| 142 | def group_fstypes(self, image_fstypes): | ||
| 143 | self.graph = self._construct_dep_graph(image_fstypes) | ||
| 144 | |||
| 145 | self._clean_graph() | ||
| 146 | |||
| 147 | self.deps_array = self._compute_dependencies() | ||
| 148 | |||
| 149 | alltypes = [node for node in self.graph] | ||
| 150 | |||
| 151 | return (alltypes, self._sort_graph()) | ||
| 152 | |||
| 153 | |||
| 154 | class Image(ImageDepGraph): | ||
| 155 | def __init__(self, d): | ||
| 156 | self.d = d | ||
| 157 | |||
| 158 | super(Image, self).__init__(d) | ||
| 159 | |||
| 160 | def _get_rootfs_size(self): | ||
| 161 | """compute the rootfs size""" | ||
| 162 | rootfs_alignment = int(self.d.getVar('IMAGE_ROOTFS_ALIGNMENT', True)) | ||
| 163 | overhead_factor = float(self.d.getVar('IMAGE_OVERHEAD_FACTOR', True)) | ||
| 164 | rootfs_req_size = int(self.d.getVar('IMAGE_ROOTFS_SIZE', True)) | ||
| 165 | rootfs_extra_space = eval(self.d.getVar('IMAGE_ROOTFS_EXTRA_SPACE', True)) | ||
| 166 | rootfs_maxsize = self.d.getVar('IMAGE_ROOTFS_MAXSIZE', True) | ||
| 167 | |||
| 168 | output = subprocess.check_output(['du', '-ks', | ||
| 169 | self.d.getVar('IMAGE_ROOTFS', True)]) | ||
| 170 | size_kb = int(output.split()[0]) | ||
| 171 | base_size = size_kb * overhead_factor | ||
| 172 | base_size = (base_size, rootfs_req_size)[base_size < rootfs_req_size] + \ | ||
| 173 | rootfs_extra_space | ||
| 174 | |||
| 175 | if base_size != int(base_size): | ||
| 176 | base_size = int(base_size + 1) | ||
| 177 | else: | ||
| 178 | base_size = int(base_size) | ||
| 179 | |||
| 180 | base_size += rootfs_alignment - 1 | ||
| 181 | base_size -= base_size % rootfs_alignment | ||
| 182 | |||
| 183 | # Check the rootfs size against IMAGE_ROOTFS_MAXSIZE (if set) | ||
| 184 | if rootfs_maxsize: | ||
| 185 | rootfs_maxsize_int = int(rootfs_maxsize) | ||
| 186 | if base_size > rootfs_maxsize_int: | ||
| 187 | bb.fatal("The rootfs size %d(K) overrides the max size %d(K)" % \ | ||
| 188 | (base_size, rootfs_maxsize_int)) | ||
| 189 | |||
| 190 | return base_size | ||
| 191 | |||
| 192 | def _create_symlinks(self, subimages): | ||
| 193 | """create symlinks to the newly created image""" | ||
| 194 | deploy_dir = self.d.getVar('DEPLOY_DIR_IMAGE', True) | ||
| 195 | img_name = self.d.getVar('IMAGE_NAME', True) | ||
| 196 | link_name = self.d.getVar('IMAGE_LINK_NAME', True) | ||
| 197 | manifest_name = self.d.getVar('IMAGE_MANIFEST', True) | ||
| 198 | |||
| 199 | os.chdir(deploy_dir) | ||
| 200 | |||
| 201 | if link_name: | ||
| 202 | for type in subimages: | ||
| 203 | if os.path.exists(img_name + ".rootfs." + type): | ||
| 204 | dst = link_name + "." + type | ||
| 205 | src = img_name + ".rootfs." + type | ||
| 206 | bb.note("Creating symlink: %s -> %s" % (dst, src)) | ||
| 207 | os.symlink(src, dst) | ||
| 208 | |||
| 209 | if manifest_name is not None and \ | ||
| 210 | os.path.exists(manifest_name) and \ | ||
| 211 | not os.path.exists(link_name + ".manifest"): | ||
| 212 | os.symlink(os.path.basename(manifest_name), | ||
| 213 | link_name + ".manifest") | ||
| 214 | |||
| 215 | def _remove_old_symlinks(self): | ||
| 216 | """remove the symlinks to old binaries""" | ||
| 217 | |||
| 218 | if self.d.getVar('IMAGE_LINK_NAME', True): | ||
| 219 | deploy_dir = self.d.getVar('DEPLOY_DIR_IMAGE', True) | ||
| 220 | for img in os.listdir(deploy_dir): | ||
| 221 | if img.find(self.d.getVar('IMAGE_LINK_NAME', True)) == 0: | ||
| 222 | img = os.path.join(deploy_dir, img) | ||
| 223 | if os.path.islink(img): | ||
| 224 | if self.d.getVar('RM_OLD_IMAGE', True) == "1" and \ | ||
| 225 | os.path.exists(os.path.realpath(img)): | ||
| 226 | os.remove(os.path.realpath(img)) | ||
| 227 | |||
| 228 | os.remove(img) | ||
| 229 | |||
| 230 | """ | ||
| 231 | This function will just filter out the compressed image types from the | ||
| 232 | fstype groups returning a (filtered_fstype_groups, cimages) tuple. | ||
| 233 | """ | ||
| 234 | def _filter_out_commpressed(self, fstype_groups): | ||
| 235 | ctypes = self.d.getVar('COMPRESSIONTYPES', True).split() | ||
| 236 | cimages = {} | ||
| 237 | |||
| 238 | filtered_groups = [] | ||
| 239 | for group in fstype_groups: | ||
| 240 | filtered_group = [] | ||
| 241 | for type in group: | ||
| 242 | basetype = None | ||
| 243 | for ctype in ctypes: | ||
| 244 | if type.endswith("." + ctype): | ||
| 245 | basetype = type[:-len("." + ctype)] | ||
| 246 | if basetype not in filtered_group: | ||
| 247 | filtered_group.append(basetype) | ||
| 248 | if basetype not in cimages: | ||
| 249 | cimages[basetype] = [] | ||
| 250 | if ctype not in cimages[basetype]: | ||
| 251 | cimages[basetype].append(ctype) | ||
| 252 | break | ||
| 253 | if not basetype and type not in filtered_group: | ||
| 254 | filtered_group.append(type) | ||
| 255 | |||
| 256 | filtered_groups.append(filtered_group) | ||
| 257 | |||
| 258 | return (filtered_groups, cimages) | ||
| 259 | |||
| 260 | def _get_image_types(self): | ||
| 261 | """returns a (types, cimages) tuple""" | ||
| 262 | |||
| 263 | alltypes, fstype_groups = self.group_fstypes(self.d.getVar('IMAGE_FSTYPES', True).split()) | ||
| 264 | |||
| 265 | filtered_groups, cimages = self._filter_out_commpressed(fstype_groups) | ||
| 266 | |||
| 267 | return (alltypes, filtered_groups, cimages) | ||
| 268 | |||
| 269 | def _write_script(self, type, cmds, sprefix=""): | ||
| 270 | tempdir = self.d.getVar('T', True) | ||
| 271 | script_name = os.path.join(tempdir, sprefix + "create_image." + type) | ||
| 272 | rootfs_size = self._get_rootfs_size() | ||
| 273 | |||
| 274 | self.d.setVar('img_creation_func', '\n'.join(cmds)) | ||
| 275 | self.d.setVarFlag('img_creation_func', 'func', '1') | ||
| 276 | self.d.setVarFlag('img_creation_func', 'fakeroot', '1') | ||
| 277 | self.d.setVar('ROOTFS_SIZE', str(rootfs_size)) | ||
| 278 | |||
| 279 | with open(script_name, "w+") as script: | ||
| 280 | script.write("%s" % bb.build.shell_trap_code()) | ||
| 281 | script.write("export ROOTFS_SIZE=%d\n" % rootfs_size) | ||
| 282 | bb.data.emit_func('img_creation_func', script, self.d) | ||
| 283 | script.write("img_creation_func\n") | ||
| 284 | |||
| 285 | os.chmod(script_name, 0775) | ||
| 286 | |||
| 287 | return script_name | ||
| 288 | |||
| 289 | def _get_imagecmds(self, sprefix=""): | ||
| 290 | old_overrides = self.d.getVar('OVERRIDES', 0) | ||
| 291 | |||
| 292 | alltypes, fstype_groups, cimages = self._get_image_types() | ||
| 293 | |||
| 294 | image_cmd_groups = [] | ||
| 295 | |||
| 296 | bb.note("The image creation groups are: %s" % str(fstype_groups)) | ||
| 297 | for fstype_group in fstype_groups: | ||
| 298 | image_cmds = [] | ||
| 299 | for type in fstype_group: | ||
| 300 | cmds = [] | ||
| 301 | subimages = [] | ||
| 302 | |||
| 303 | localdata = bb.data.createCopy(self.d) | ||
| 304 | localdata.setVar('OVERRIDES', '%s:%s' % (type, old_overrides)) | ||
| 305 | bb.data.update_data(localdata) | ||
| 306 | localdata.setVar('type', type) | ||
| 307 | |||
| 308 | image_cmd = localdata.getVar("IMAGE_CMD", True) | ||
| 309 | if image_cmd: | ||
| 310 | cmds.append("\t" + image_cmd) | ||
| 311 | else: | ||
| 312 | bb.fatal("No IMAGE_CMD defined for IMAGE_FSTYPES entry '%s' - possibly invalid type name or missing support class" % type) | ||
| 313 | cmds.append(localdata.expand("\tcd ${DEPLOY_DIR_IMAGE}")) | ||
| 314 | |||
| 315 | if type in cimages: | ||
| 316 | for ctype in cimages[type]: | ||
| 317 | cmds.append("\t" + localdata.getVar("COMPRESS_CMD_" + ctype, True)) | ||
| 318 | subimages.append(type + "." + ctype) | ||
| 319 | |||
| 320 | if type not in alltypes: | ||
| 321 | cmds.append(localdata.expand("\trm ${IMAGE_NAME}.rootfs.${type}")) | ||
| 322 | else: | ||
| 323 | subimages.append(type) | ||
| 324 | |||
| 325 | script_name = self._write_script(type, cmds, sprefix) | ||
| 326 | |||
| 327 | image_cmds.append((type, subimages, script_name, sprefix)) | ||
| 328 | |||
| 329 | image_cmd_groups.append(image_cmds) | ||
| 330 | |||
| 331 | return image_cmd_groups | ||
| 332 | |||
| 333 | def _write_wic_env(self): | ||
| 334 | """ | ||
| 335 | Write environment variables used by wic | ||
| 336 | to tmp/sysroots/<machine>/imgdata/<image>.env | ||
| 337 | """ | ||
| 338 | wicvars = self.d.getVar('WICVARS', True) | ||
| 339 | if not wicvars: | ||
| 340 | return | ||
| 341 | |||
| 342 | stdir = self.d.getVar('STAGING_DIR_TARGET', True) | ||
| 343 | outdir = os.path.join(stdir, 'imgdata') | ||
| 344 | bb.utils.mkdirhier(outdir) | ||
| 345 | basename = self.d.getVar('IMAGE_BASENAME', True) | ||
| 346 | with open(os.path.join(outdir, basename) + '.env', 'w') as envf: | ||
| 347 | for var in wicvars.split(): | ||
| 348 | value = self.d.getVar(var, True) | ||
| 349 | if value: | ||
| 350 | envf.write('%s="%s"\n' % (var, value.strip())) | ||
| 351 | |||
| 352 | def create(self): | ||
| 353 | bb.note("###### Generate images #######") | ||
| 354 | |||
| 355 | image_cmd_groups = self._get_imagecmds() | ||
| 356 | |||
| 357 | # Process the debug filesystem... | ||
| 358 | debugfs_d = bb.data.createCopy(self.d) | ||
| 359 | if self.d.getVar('IMAGE_GEN_DEBUGFS', True) == "1": | ||
| 360 | bb.note("Processing debugfs image(s) ...") | ||
| 361 | orig_d = self.d | ||
| 362 | self.d = debugfs_d | ||
| 363 | |||
| 364 | self.d.setVar('IMAGE_ROOTFS', orig_d.getVar('IMAGE_ROOTFS', True) + '-dbg') | ||
| 365 | self.d.setVar('IMAGE_NAME', orig_d.getVar('IMAGE_NAME', True) + '-dbg') | ||
| 366 | self.d.setVar('IMAGE_LINK_NAME', orig_d.getVar('IMAGE_LINK_NAME', True) + '-dbg') | ||
| 367 | |||
| 368 | debugfs_image_fstypes = orig_d.getVar('IMAGE_FSTYPES_DEBUGFS', True) | ||
| 369 | if debugfs_image_fstypes: | ||
| 370 | self.d.setVar('IMAGE_FSTYPES', orig_d.getVar('IMAGE_FSTYPES_DEBUGFS', True)) | ||
| 371 | |||
| 372 | self._remove_old_symlinks() | ||
| 373 | |||
| 374 | image_cmd_groups += self._get_imagecmds("debugfs.") | ||
| 375 | |||
| 376 | self.d = orig_d | ||
| 377 | |||
| 378 | self._write_wic_env() | ||
| 379 | |||
| 380 | for image_cmds in image_cmd_groups: | ||
| 381 | # create the images in parallel | ||
| 382 | nproc = multiprocessing.cpu_count() | ||
| 383 | pool = bb.utils.multiprocessingpool(nproc) | ||
| 384 | results = list(pool.imap(generate_image, image_cmds)) | ||
| 385 | pool.close() | ||
| 386 | pool.join() | ||
| 387 | |||
| 388 | for result in results: | ||
| 389 | if result is not None: | ||
| 390 | bb.fatal(result) | ||
| 391 | |||
| 392 | for image_type, subimages, script, sprefix in image_cmds: | ||
| 393 | if sprefix == 'debugfs.': | ||
| 394 | bb.note("Creating symlinks for %s debugfs image ..." % image_type) | ||
| 395 | orig_d = self.d | ||
| 396 | self.d = debugfs_d | ||
| 397 | self._create_symlinks(subimages) | ||
| 398 | self.d = orig_d | ||
| 399 | else: | ||
| 400 | bb.note("Creating symlinks for %s image ..." % image_type) | ||
| 401 | self._create_symlinks(subimages) | ||
| 402 | |||
| 403 | def create_image(d): | ||
| 404 | Image(d).create() | ||
| 405 | |||
| 406 | if __name__ == "__main__": | ||
| 407 | """ | ||
| 408 | Image creation can be called independent from bitbake environment. | ||
| 409 | """ | ||
| 410 | """ | ||
| 411 | TBD | ||
| 412 | """ | ||
