diff options
Diffstat (limited to 'scripts/lib/mic/utils/BmapCreate.py')
-rw-r--r-- | scripts/lib/mic/utils/BmapCreate.py | 298 |
1 files changed, 0 insertions, 298 deletions
diff --git a/scripts/lib/mic/utils/BmapCreate.py b/scripts/lib/mic/utils/BmapCreate.py deleted file mode 100644 index 65b19a5f46..0000000000 --- a/scripts/lib/mic/utils/BmapCreate.py +++ /dev/null | |||
@@ -1,298 +0,0 @@ | |||
1 | """ This module implements the block map (bmap) creation functionality and | ||
2 | provides the corresponding API in form of the 'BmapCreate' class. | ||
3 | |||
4 | The idea is that while images files may generally be very large (e.g., 4GiB), | ||
5 | they may nevertheless contain only little real data, e.g., 512MiB. This data | ||
6 | are files, directories, file-system meta-data, partition table, etc. When | ||
7 | copying the image to the target device, you do not have to copy all the 4GiB of | ||
8 | data, you can copy only 512MiB of it, which is 4 times less, so copying should | ||
9 | presumably be 4 times faster. | ||
10 | |||
11 | The block map file is an XML file which contains a list of blocks which have to | ||
12 | be copied to the target device. The other blocks are not used and there is no | ||
13 | need to copy them. The XML file also contains some additional information like | ||
14 | block size, image size, count of mapped blocks, etc. There are also many | ||
15 | commentaries, so it is human-readable. | ||
16 | |||
17 | The image has to be a sparse file. Generally, this means that when you generate | ||
18 | this image file, you should start with a huge sparse file which contains a | ||
19 | single hole spanning the entire file. Then you should partition it, write all | ||
20 | the data (probably by means of loop-back mounting the image or parts of it), | ||
21 | etc. The end result should be a sparse file where mapped areas represent useful | ||
22 | parts of the image and holes represent useless parts of the image, which do not | ||
23 | have to be copied when copying the image to the target device. | ||
24 | |||
25 | This module uses the FIBMAP ioctl to detect holes. """ | ||
26 | |||
27 | # Disable the following pylint recommendations: | ||
28 | # * Too many instance attributes - R0902 | ||
29 | # * Too few public methods - R0903 | ||
30 | # pylint: disable=R0902,R0903 | ||
31 | |||
32 | import hashlib | ||
33 | from mic.utils.misc import human_size | ||
34 | from mic.utils import Fiemap | ||
35 | |||
36 | # The bmap format version we generate | ||
37 | SUPPORTED_BMAP_VERSION = "1.3" | ||
38 | |||
39 | _BMAP_START_TEMPLATE = \ | ||
40 | """<?xml version="1.0" ?> | ||
41 | <!-- This file contains the block map for an image file, which is basically | ||
42 | a list of useful (mapped) block numbers in the image file. In other words, | ||
43 | it lists only those blocks which contain data (boot sector, partition | ||
44 | table, file-system metadata, files, directories, extents, etc). These | ||
45 | blocks have to be copied to the target device. The other blocks do not | ||
46 | contain any useful data and do not have to be copied to the target | ||
47 | device. | ||
48 | |||
49 | The block map an optimization which allows to copy or flash the image to | ||
50 | the image quicker than copying of flashing the entire image. This is | ||
51 | because with bmap less data is copied: <MappedBlocksCount> blocks instead | ||
52 | of <BlocksCount> blocks. | ||
53 | |||
54 | Besides the machine-readable data, this file contains useful commentaries | ||
55 | which contain human-readable information like image size, percentage of | ||
56 | mapped data, etc. | ||
57 | |||
58 | The 'version' attribute is the block map file format version in the | ||
59 | 'major.minor' format. The version major number is increased whenever an | ||
60 | incompatible block map format change is made. The minor number changes | ||
61 | in case of minor backward-compatible changes. --> | ||
62 | |||
63 | <bmap version="%s"> | ||
64 | <!-- Image size in bytes: %s --> | ||
65 | <ImageSize> %u </ImageSize> | ||
66 | |||
67 | <!-- Size of a block in bytes --> | ||
68 | <BlockSize> %u </BlockSize> | ||
69 | |||
70 | <!-- Count of blocks in the image file --> | ||
71 | <BlocksCount> %u </BlocksCount> | ||
72 | |||
73 | """ | ||
74 | |||
75 | class Error(Exception): | ||
76 | """ A class for exceptions generated by this module. We currently support | ||
77 | only one type of exceptions, and we basically throw human-readable problem | ||
78 | description in case of errors. """ | ||
79 | pass | ||
80 | |||
81 | class BmapCreate: | ||
82 | """ This class implements the bmap creation functionality. To generate a | ||
83 | bmap for an image (which is supposedly a sparse file), you should first | ||
84 | create an instance of 'BmapCreate' and provide: | ||
85 | |||
86 | * full path or a file-like object of the image to create bmap for | ||
87 | * full path or a file object to use for writing the results to | ||
88 | |||
89 | Then you should invoke the 'generate()' method of this class. It will use | ||
90 | the FIEMAP ioctl to generate the bmap. """ | ||
91 | |||
92 | def _open_image_file(self): | ||
93 | """ Open the image file. """ | ||
94 | |||
95 | try: | ||
96 | self._f_image = open(self._image_path, 'rb') | ||
97 | except IOError as err: | ||
98 | raise Error("cannot open image file '%s': %s" \ | ||
99 | % (self._image_path, err)) | ||
100 | |||
101 | self._f_image_needs_close = True | ||
102 | |||
103 | def _open_bmap_file(self): | ||
104 | """ Open the bmap file. """ | ||
105 | |||
106 | try: | ||
107 | self._f_bmap = open(self._bmap_path, 'w+') | ||
108 | except IOError as err: | ||
109 | raise Error("cannot open bmap file '%s': %s" \ | ||
110 | % (self._bmap_path, err)) | ||
111 | |||
112 | self._f_bmap_needs_close = True | ||
113 | |||
114 | def __init__(self, image, bmap): | ||
115 | """ Initialize a class instance: | ||
116 | * image - full path or a file-like object of the image to create bmap | ||
117 | for | ||
118 | * bmap - full path or a file object to use for writing the resulting | ||
119 | bmap to """ | ||
120 | |||
121 | self.image_size = None | ||
122 | self.image_size_human = None | ||
123 | self.block_size = None | ||
124 | self.blocks_cnt = None | ||
125 | self.mapped_cnt = None | ||
126 | self.mapped_size = None | ||
127 | self.mapped_size_human = None | ||
128 | self.mapped_percent = None | ||
129 | |||
130 | self._mapped_count_pos1 = None | ||
131 | self._mapped_count_pos2 = None | ||
132 | self._sha1_pos = None | ||
133 | |||
134 | self._f_image_needs_close = False | ||
135 | self._f_bmap_needs_close = False | ||
136 | |||
137 | if hasattr(image, "read"): | ||
138 | self._f_image = image | ||
139 | self._image_path = image.name | ||
140 | else: | ||
141 | self._image_path = image | ||
142 | self._open_image_file() | ||
143 | |||
144 | if hasattr(bmap, "read"): | ||
145 | self._f_bmap = bmap | ||
146 | self._bmap_path = bmap.name | ||
147 | else: | ||
148 | self._bmap_path = bmap | ||
149 | self._open_bmap_file() | ||
150 | |||
151 | self.fiemap = Fiemap.Fiemap(self._f_image) | ||
152 | |||
153 | self.image_size = self.fiemap.image_size | ||
154 | self.image_size_human = human_size(self.image_size) | ||
155 | if self.image_size == 0: | ||
156 | raise Error("cannot generate bmap for zero-sized image file '%s'" \ | ||
157 | % self._image_path) | ||
158 | |||
159 | self.block_size = self.fiemap.block_size | ||
160 | self.blocks_cnt = self.fiemap.blocks_cnt | ||
161 | |||
162 | def _bmap_file_start(self): | ||
163 | """ A helper function which generates the starting contents of the | ||
164 | block map file: the header comment, image size, block size, etc. """ | ||
165 | |||
166 | # We do not know the amount of mapped blocks at the moment, so just put | ||
167 | # whitespaces instead of real numbers. Assume the longest possible | ||
168 | # numbers. | ||
169 | mapped_count = ' ' * len(str(self.image_size)) | ||
170 | mapped_size_human = ' ' * len(self.image_size_human) | ||
171 | |||
172 | xml = _BMAP_START_TEMPLATE \ | ||
173 | % (SUPPORTED_BMAP_VERSION, self.image_size_human, | ||
174 | self.image_size, self.block_size, self.blocks_cnt) | ||
175 | xml += " <!-- Count of mapped blocks: " | ||
176 | |||
177 | self._f_bmap.write(xml) | ||
178 | self._mapped_count_pos1 = self._f_bmap.tell() | ||
179 | |||
180 | # Just put white-spaces instead of real information about mapped blocks | ||
181 | xml = "%s or %.1f -->\n" % (mapped_size_human, 100.0) | ||
182 | xml += " <MappedBlocksCount> " | ||
183 | |||
184 | self._f_bmap.write(xml) | ||
185 | self._mapped_count_pos2 = self._f_bmap.tell() | ||
186 | |||
187 | xml = "%s </MappedBlocksCount>\n\n" % mapped_count | ||
188 | |||
189 | # pylint: disable=C0301 | ||
190 | xml += " <!-- The checksum of this bmap file. When it is calculated, the value of\n" | ||
191 | xml += " the SHA1 checksum has be zeoro (40 ASCII \"0\" symbols). -->\n" | ||
192 | xml += " <BmapFileSHA1> " | ||
193 | |||
194 | self._f_bmap.write(xml) | ||
195 | self._sha1_pos = self._f_bmap.tell() | ||
196 | |||
197 | xml = "0" * 40 + " </BmapFileSHA1>\n\n" | ||
198 | xml += " <!-- The block map which consists of elements which may either be a\n" | ||
199 | xml += " range of blocks or a single block. The 'sha1' attribute (if present)\n" | ||
200 | xml += " is the SHA1 checksum of this blocks range. -->\n" | ||
201 | xml += " <BlockMap>\n" | ||
202 | # pylint: enable=C0301 | ||
203 | |||
204 | self._f_bmap.write(xml) | ||
205 | |||
206 | def _bmap_file_end(self): | ||
207 | """ A helper function which generates the final parts of the block map | ||
208 | file: the ending tags and the information about the amount of mapped | ||
209 | blocks. """ | ||
210 | |||
211 | xml = " </BlockMap>\n" | ||
212 | xml += "</bmap>\n" | ||
213 | |||
214 | self._f_bmap.write(xml) | ||
215 | |||
216 | self._f_bmap.seek(self._mapped_count_pos1) | ||
217 | self._f_bmap.write("%s or %.1f%%" % \ | ||
218 | (self.mapped_size_human, self.mapped_percent)) | ||
219 | |||
220 | self._f_bmap.seek(self._mapped_count_pos2) | ||
221 | self._f_bmap.write("%u" % self.mapped_cnt) | ||
222 | |||
223 | self._f_bmap.seek(0) | ||
224 | sha1 = hashlib.sha1(self._f_bmap.read()).hexdigest() | ||
225 | self._f_bmap.seek(self._sha1_pos) | ||
226 | self._f_bmap.write("%s" % sha1) | ||
227 | |||
228 | def _calculate_sha1(self, first, last): | ||
229 | """ A helper function which calculates SHA1 checksum for the range of | ||
230 | blocks of the image file: from block 'first' to block 'last'. """ | ||
231 | |||
232 | start = first * self.block_size | ||
233 | end = (last + 1) * self.block_size | ||
234 | |||
235 | self._f_image.seek(start) | ||
236 | hash_obj = hashlib.new("sha1") | ||
237 | |||
238 | chunk_size = 1024*1024 | ||
239 | to_read = end - start | ||
240 | read = 0 | ||
241 | |||
242 | while read < to_read: | ||
243 | if read + chunk_size > to_read: | ||
244 | chunk_size = to_read - read | ||
245 | chunk = self._f_image.read(chunk_size) | ||
246 | hash_obj.update(chunk) | ||
247 | read += chunk_size | ||
248 | |||
249 | return hash_obj.hexdigest() | ||
250 | |||
251 | def generate(self, include_checksums = True): | ||
252 | """ Generate bmap for the image file. If 'include_checksums' is 'True', | ||
253 | also generate SHA1 checksums for block ranges. """ | ||
254 | |||
255 | # Save image file position in order to restore it at the end | ||
256 | image_pos = self._f_image.tell() | ||
257 | |||
258 | self._bmap_file_start() | ||
259 | |||
260 | # Generate the block map and write it to the XML block map | ||
261 | # file as we go. | ||
262 | self.mapped_cnt = 0 | ||
263 | for first, last in self.fiemap.get_mapped_ranges(0, self.blocks_cnt): | ||
264 | self.mapped_cnt += last - first + 1 | ||
265 | if include_checksums: | ||
266 | sha1 = self._calculate_sha1(first, last) | ||
267 | sha1 = " sha1=\"%s\"" % sha1 | ||
268 | else: | ||
269 | sha1 = "" | ||
270 | |||
271 | if first != last: | ||
272 | self._f_bmap.write(" <Range%s> %s-%s </Range>\n" \ | ||
273 | % (sha1, first, last)) | ||
274 | else: | ||
275 | self._f_bmap.write(" <Range%s> %s </Range>\n" \ | ||
276 | % (sha1, first)) | ||
277 | |||
278 | self.mapped_size = self.mapped_cnt * self.block_size | ||
279 | self.mapped_size_human = human_size(self.mapped_size) | ||
280 | self.mapped_percent = (self.mapped_cnt * 100.0) / self.blocks_cnt | ||
281 | |||
282 | self._bmap_file_end() | ||
283 | |||
284 | try: | ||
285 | self._f_bmap.flush() | ||
286 | except IOError as err: | ||
287 | raise Error("cannot flush the bmap file '%s': %s" \ | ||
288 | % (self._bmap_path, err)) | ||
289 | |||
290 | self._f_image.seek(image_pos) | ||
291 | |||
292 | def __del__(self): | ||
293 | """ The class destructor which closes the opened files. """ | ||
294 | |||
295 | if self._f_image_needs_close: | ||
296 | self._f_image.close() | ||
297 | if self._f_bmap_needs_close: | ||
298 | self._f_bmap.close() | ||