From 9fc88f96d40b17c90bac53b90045a87b2d2cff84 Mon Sep 17 00:00:00 2001 From: Tom Zanussi Date: Sat, 24 Aug 2013 15:31:34 +0000 Subject: wic: Add mic w/pykickstart This is the starting point for the implemention described in [YOCTO 3847] which came to the conclusion that it would make sense to use kickstart syntax to implement image creation in OpenEmbedded. I subsequently realized that there was an existing tool that already implemented image creation using kickstart syntax, the Tizen/Meego mic tool. As such, it made sense to use that as a starting point - this commit essentially just copies the relevant Python code from the MIC tool to the scripts/lib dir, where it can be accessed by the previously created wic tool. Most of this will be removed or renamed by later commits, since we're initially focusing on partitioning only. Care should be taken so that we can easily add back any additional functionality should we decide later to expand the tool, though (we may also want to contribute our local changes to the mic tool to the Tizen project if it makes sense, and therefore should avoid gratuitous changes to the original code if possible). Added the /mic subdir from Tizen mic repo as a starting point: git clone git://review.tizen.org/tools/mic.git For reference, the top commit: commit 20164175ddc234a17b8a12c33d04b012347b1530 Author: Gui Chen Date: Sun Jun 30 22:32:16 2013 -0400 bump up to 0.19.2 Also added the /plugins subdir, moved to under the /mic subdir (to match the default plugin_dir location in mic.conf.in, which was renamed to yocto-image.conf (moved and renamed by later patches) and put into /scripts. (From OE-Core rev: 31f0360f1fd4ebc9dfcaed42d1c50d2448b4632e) Signed-off-by: Tom Zanussi Signed-off-by: Saul Wold Signed-off-by: Richard Purdie --- scripts/lib/mic/utils/BmapCreate.py | 298 ++++++++++++++++++++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 scripts/lib/mic/utils/BmapCreate.py (limited to 'scripts/lib/mic/utils/BmapCreate.py') diff --git a/scripts/lib/mic/utils/BmapCreate.py b/scripts/lib/mic/utils/BmapCreate.py new file mode 100644 index 0000000000..65b19a5f46 --- /dev/null +++ b/scripts/lib/mic/utils/BmapCreate.py @@ -0,0 +1,298 @@ +""" This module implements the block map (bmap) creation functionality and +provides the corresponding API in form of the 'BmapCreate' class. + +The idea is that while images files may generally be very large (e.g., 4GiB), +they may nevertheless contain only little real data, e.g., 512MiB. This data +are files, directories, file-system meta-data, partition table, etc. When +copying the image to the target device, you do not have to copy all the 4GiB of +data, you can copy only 512MiB of it, which is 4 times less, so copying should +presumably be 4 times faster. + +The block map file is an XML file which contains a list of blocks which have to +be copied to the target device. The other blocks are not used and there is no +need to copy them. The XML file also contains some additional information like +block size, image size, count of mapped blocks, etc. There are also many +commentaries, so it is human-readable. + +The image has to be a sparse file. Generally, this means that when you generate +this image file, you should start with a huge sparse file which contains a +single hole spanning the entire file. Then you should partition it, write all +the data (probably by means of loop-back mounting the image or parts of it), +etc. The end result should be a sparse file where mapped areas represent useful +parts of the image and holes represent useless parts of the image, which do not +have to be copied when copying the image to the target device. + +This module uses the FIBMAP ioctl to detect holes. """ + +# Disable the following pylint recommendations: +# * Too many instance attributes - R0902 +# * Too few public methods - R0903 +# pylint: disable=R0902,R0903 + +import hashlib +from mic.utils.misc import human_size +from mic.utils import Fiemap + +# The bmap format version we generate +SUPPORTED_BMAP_VERSION = "1.3" + +_BMAP_START_TEMPLATE = \ +""" + + + + + %u + + + %u + + + %u + +""" + +class Error(Exception): + """ A class for exceptions generated by this module. We currently support + only one type of exceptions, and we basically throw human-readable problem + description in case of errors. """ + pass + +class BmapCreate: + """ This class implements the bmap creation functionality. To generate a + bmap for an image (which is supposedly a sparse file), you should first + create an instance of 'BmapCreate' and provide: + + * full path or a file-like object of the image to create bmap for + * full path or a file object to use for writing the results to + + Then you should invoke the 'generate()' method of this class. It will use + the FIEMAP ioctl to generate the bmap. """ + + def _open_image_file(self): + """ Open the image file. """ + + try: + self._f_image = open(self._image_path, 'rb') + except IOError as err: + raise Error("cannot open image file '%s': %s" \ + % (self._image_path, err)) + + self._f_image_needs_close = True + + def _open_bmap_file(self): + """ Open the bmap file. """ + + try: + self._f_bmap = open(self._bmap_path, 'w+') + except IOError as err: + raise Error("cannot open bmap file '%s': %s" \ + % (self._bmap_path, err)) + + self._f_bmap_needs_close = True + + def __init__(self, image, bmap): + """ Initialize a class instance: + * image - full path or a file-like object of the image to create bmap + for + * bmap - full path or a file object to use for writing the resulting + bmap to """ + + self.image_size = None + self.image_size_human = None + self.block_size = None + self.blocks_cnt = None + self.mapped_cnt = None + self.mapped_size = None + self.mapped_size_human = None + self.mapped_percent = None + + self._mapped_count_pos1 = None + self._mapped_count_pos2 = None + self._sha1_pos = None + + self._f_image_needs_close = False + self._f_bmap_needs_close = False + + if hasattr(image, "read"): + self._f_image = image + self._image_path = image.name + else: + self._image_path = image + self._open_image_file() + + if hasattr(bmap, "read"): + self._f_bmap = bmap + self._bmap_path = bmap.name + else: + self._bmap_path = bmap + self._open_bmap_file() + + self.fiemap = Fiemap.Fiemap(self._f_image) + + self.image_size = self.fiemap.image_size + self.image_size_human = human_size(self.image_size) + if self.image_size == 0: + raise Error("cannot generate bmap for zero-sized image file '%s'" \ + % self._image_path) + + self.block_size = self.fiemap.block_size + self.blocks_cnt = self.fiemap.blocks_cnt + + def _bmap_file_start(self): + """ A helper function which generates the starting contents of the + block map file: the header comment, image size, block size, etc. """ + + # We do not know the amount of mapped blocks at the moment, so just put + # whitespaces instead of real numbers. Assume the longest possible + # numbers. + mapped_count = ' ' * len(str(self.image_size)) + mapped_size_human = ' ' * len(self.image_size_human) + + xml = _BMAP_START_TEMPLATE \ + % (SUPPORTED_BMAP_VERSION, self.image_size_human, + self.image_size, self.block_size, self.blocks_cnt) + xml += " \n" % (mapped_size_human, 100.0) + xml += " " + + self._f_bmap.write(xml) + self._mapped_count_pos2 = self._f_bmap.tell() + + xml = "%s \n\n" % mapped_count + + # pylint: disable=C0301 + xml += " \n" + xml += " " + + self._f_bmap.write(xml) + self._sha1_pos = self._f_bmap.tell() + + xml = "0" * 40 + " \n\n" + xml += " \n" + xml += " \n" + # pylint: enable=C0301 + + self._f_bmap.write(xml) + + def _bmap_file_end(self): + """ A helper function which generates the final parts of the block map + file: the ending tags and the information about the amount of mapped + blocks. """ + + xml = " \n" + xml += "\n" + + self._f_bmap.write(xml) + + self._f_bmap.seek(self._mapped_count_pos1) + self._f_bmap.write("%s or %.1f%%" % \ + (self.mapped_size_human, self.mapped_percent)) + + self._f_bmap.seek(self._mapped_count_pos2) + self._f_bmap.write("%u" % self.mapped_cnt) + + self._f_bmap.seek(0) + sha1 = hashlib.sha1(self._f_bmap.read()).hexdigest() + self._f_bmap.seek(self._sha1_pos) + self._f_bmap.write("%s" % sha1) + + def _calculate_sha1(self, first, last): + """ A helper function which calculates SHA1 checksum for the range of + blocks of the image file: from block 'first' to block 'last'. """ + + start = first * self.block_size + end = (last + 1) * self.block_size + + self._f_image.seek(start) + hash_obj = hashlib.new("sha1") + + chunk_size = 1024*1024 + to_read = end - start + read = 0 + + while read < to_read: + if read + chunk_size > to_read: + chunk_size = to_read - read + chunk = self._f_image.read(chunk_size) + hash_obj.update(chunk) + read += chunk_size + + return hash_obj.hexdigest() + + def generate(self, include_checksums = True): + """ Generate bmap for the image file. If 'include_checksums' is 'True', + also generate SHA1 checksums for block ranges. """ + + # Save image file position in order to restore it at the end + image_pos = self._f_image.tell() + + self._bmap_file_start() + + # Generate the block map and write it to the XML block map + # file as we go. + self.mapped_cnt = 0 + for first, last in self.fiemap.get_mapped_ranges(0, self.blocks_cnt): + self.mapped_cnt += last - first + 1 + if include_checksums: + sha1 = self._calculate_sha1(first, last) + sha1 = " sha1=\"%s\"" % sha1 + else: + sha1 = "" + + if first != last: + self._f_bmap.write(" %s-%s \n" \ + % (sha1, first, last)) + else: + self._f_bmap.write(" %s \n" \ + % (sha1, first)) + + self.mapped_size = self.mapped_cnt * self.block_size + self.mapped_size_human = human_size(self.mapped_size) + self.mapped_percent = (self.mapped_cnt * 100.0) / self.blocks_cnt + + self._bmap_file_end() + + try: + self._f_bmap.flush() + except IOError as err: + raise Error("cannot flush the bmap file '%s': %s" \ + % (self._bmap_path, err)) + + self._f_image.seek(image_pos) + + def __del__(self): + """ The class destructor which closes the opened files. """ + + if self._f_image_needs_close: + self._f_image.close() + if self._f_bmap_needs_close: + self._f_bmap.close() -- cgit v1.2.3-54-g00ecf