summaryrefslogtreecommitdiffstats
path: root/scripts/lib/mic/utils/BmapCreate.py
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/lib/mic/utils/BmapCreate.py')
-rw-r--r--scripts/lib/mic/utils/BmapCreate.py298
1 files changed, 0 insertions, 298 deletions
diff --git a/scripts/lib/mic/utils/BmapCreate.py b/scripts/lib/mic/utils/BmapCreate.py
deleted file mode 100644
index 65b19a5f46..0000000000
--- a/scripts/lib/mic/utils/BmapCreate.py
+++ /dev/null
@@ -1,298 +0,0 @@
1""" This module implements the block map (bmap) creation functionality and
2provides the corresponding API in form of the 'BmapCreate' class.
3
4The idea is that while images files may generally be very large (e.g., 4GiB),
5they may nevertheless contain only little real data, e.g., 512MiB. This data
6are files, directories, file-system meta-data, partition table, etc. When
7copying the image to the target device, you do not have to copy all the 4GiB of
8data, you can copy only 512MiB of it, which is 4 times less, so copying should
9presumably be 4 times faster.
10
11The block map file is an XML file which contains a list of blocks which have to
12be copied to the target device. The other blocks are not used and there is no
13need to copy them. The XML file also contains some additional information like
14block size, image size, count of mapped blocks, etc. There are also many
15commentaries, so it is human-readable.
16
17The image has to be a sparse file. Generally, this means that when you generate
18this image file, you should start with a huge sparse file which contains a
19single hole spanning the entire file. Then you should partition it, write all
20the data (probably by means of loop-back mounting the image or parts of it),
21etc. The end result should be a sparse file where mapped areas represent useful
22parts of the image and holes represent useless parts of the image, which do not
23have to be copied when copying the image to the target device.
24
25This module uses the FIBMAP ioctl to detect holes. """
26
27# Disable the following pylint recommendations:
28# * Too many instance attributes - R0902
29# * Too few public methods - R0903
30# pylint: disable=R0902,R0903
31
32import hashlib
33from mic.utils.misc import human_size
34from mic.utils import Fiemap
35
36# The bmap format version we generate
37SUPPORTED_BMAP_VERSION = "1.3"
38
39_BMAP_START_TEMPLATE = \
40"""<?xml version="1.0" ?>
41<!-- This file contains the block map for an image file, which is basically
42 a list of useful (mapped) block numbers in the image file. In other words,
43 it lists only those blocks which contain data (boot sector, partition
44 table, file-system metadata, files, directories, extents, etc). These
45 blocks have to be copied to the target device. The other blocks do not
46 contain any useful data and do not have to be copied to the target
47 device.
48
49 The block map an optimization which allows to copy or flash the image to
50 the image quicker than copying of flashing the entire image. This is
51 because with bmap less data is copied: <MappedBlocksCount> blocks instead
52 of <BlocksCount> blocks.
53
54 Besides the machine-readable data, this file contains useful commentaries
55 which contain human-readable information like image size, percentage of
56 mapped data, etc.
57
58 The 'version' attribute is the block map file format version in the
59 'major.minor' format. The version major number is increased whenever an
60 incompatible block map format change is made. The minor number changes
61 in case of minor backward-compatible changes. -->
62
63<bmap version="%s">
64 <!-- Image size in bytes: %s -->
65 <ImageSize> %u </ImageSize>
66
67 <!-- Size of a block in bytes -->
68 <BlockSize> %u </BlockSize>
69
70 <!-- Count of blocks in the image file -->
71 <BlocksCount> %u </BlocksCount>
72
73"""
74
75class Error(Exception):
76 """ A class for exceptions generated by this module. We currently support
77 only one type of exceptions, and we basically throw human-readable problem
78 description in case of errors. """
79 pass
80
81class BmapCreate:
82 """ This class implements the bmap creation functionality. To generate a
83 bmap for an image (which is supposedly a sparse file), you should first
84 create an instance of 'BmapCreate' and provide:
85
86 * full path or a file-like object of the image to create bmap for
87 * full path or a file object to use for writing the results to
88
89 Then you should invoke the 'generate()' method of this class. It will use
90 the FIEMAP ioctl to generate the bmap. """
91
92 def _open_image_file(self):
93 """ Open the image file. """
94
95 try:
96 self._f_image = open(self._image_path, 'rb')
97 except IOError as err:
98 raise Error("cannot open image file '%s': %s" \
99 % (self._image_path, err))
100
101 self._f_image_needs_close = True
102
103 def _open_bmap_file(self):
104 """ Open the bmap file. """
105
106 try:
107 self._f_bmap = open(self._bmap_path, 'w+')
108 except IOError as err:
109 raise Error("cannot open bmap file '%s': %s" \
110 % (self._bmap_path, err))
111
112 self._f_bmap_needs_close = True
113
114 def __init__(self, image, bmap):
115 """ Initialize a class instance:
116 * image - full path or a file-like object of the image to create bmap
117 for
118 * bmap - full path or a file object to use for writing the resulting
119 bmap to """
120
121 self.image_size = None
122 self.image_size_human = None
123 self.block_size = None
124 self.blocks_cnt = None
125 self.mapped_cnt = None
126 self.mapped_size = None
127 self.mapped_size_human = None
128 self.mapped_percent = None
129
130 self._mapped_count_pos1 = None
131 self._mapped_count_pos2 = None
132 self._sha1_pos = None
133
134 self._f_image_needs_close = False
135 self._f_bmap_needs_close = False
136
137 if hasattr(image, "read"):
138 self._f_image = image
139 self._image_path = image.name
140 else:
141 self._image_path = image
142 self._open_image_file()
143
144 if hasattr(bmap, "read"):
145 self._f_bmap = bmap
146 self._bmap_path = bmap.name
147 else:
148 self._bmap_path = bmap
149 self._open_bmap_file()
150
151 self.fiemap = Fiemap.Fiemap(self._f_image)
152
153 self.image_size = self.fiemap.image_size
154 self.image_size_human = human_size(self.image_size)
155 if self.image_size == 0:
156 raise Error("cannot generate bmap for zero-sized image file '%s'" \
157 % self._image_path)
158
159 self.block_size = self.fiemap.block_size
160 self.blocks_cnt = self.fiemap.blocks_cnt
161
162 def _bmap_file_start(self):
163 """ A helper function which generates the starting contents of the
164 block map file: the header comment, image size, block size, etc. """
165
166 # We do not know the amount of mapped blocks at the moment, so just put
167 # whitespaces instead of real numbers. Assume the longest possible
168 # numbers.
169 mapped_count = ' ' * len(str(self.image_size))
170 mapped_size_human = ' ' * len(self.image_size_human)
171
172 xml = _BMAP_START_TEMPLATE \
173 % (SUPPORTED_BMAP_VERSION, self.image_size_human,
174 self.image_size, self.block_size, self.blocks_cnt)
175 xml += " <!-- Count of mapped blocks: "
176
177 self._f_bmap.write(xml)
178 self._mapped_count_pos1 = self._f_bmap.tell()
179
180 # Just put white-spaces instead of real information about mapped blocks
181 xml = "%s or %.1f -->\n" % (mapped_size_human, 100.0)
182 xml += " <MappedBlocksCount> "
183
184 self._f_bmap.write(xml)
185 self._mapped_count_pos2 = self._f_bmap.tell()
186
187 xml = "%s </MappedBlocksCount>\n\n" % mapped_count
188
189 # pylint: disable=C0301
190 xml += " <!-- The checksum of this bmap file. When it is calculated, the value of\n"
191 xml += " the SHA1 checksum has be zeoro (40 ASCII \"0\" symbols). -->\n"
192 xml += " <BmapFileSHA1> "
193
194 self._f_bmap.write(xml)
195 self._sha1_pos = self._f_bmap.tell()
196
197 xml = "0" * 40 + " </BmapFileSHA1>\n\n"
198 xml += " <!-- The block map which consists of elements which may either be a\n"
199 xml += " range of blocks or a single block. The 'sha1' attribute (if present)\n"
200 xml += " is the SHA1 checksum of this blocks range. -->\n"
201 xml += " <BlockMap>\n"
202 # pylint: enable=C0301
203
204 self._f_bmap.write(xml)
205
206 def _bmap_file_end(self):
207 """ A helper function which generates the final parts of the block map
208 file: the ending tags and the information about the amount of mapped
209 blocks. """
210
211 xml = " </BlockMap>\n"
212 xml += "</bmap>\n"
213
214 self._f_bmap.write(xml)
215
216 self._f_bmap.seek(self._mapped_count_pos1)
217 self._f_bmap.write("%s or %.1f%%" % \
218 (self.mapped_size_human, self.mapped_percent))
219
220 self._f_bmap.seek(self._mapped_count_pos2)
221 self._f_bmap.write("%u" % self.mapped_cnt)
222
223 self._f_bmap.seek(0)
224 sha1 = hashlib.sha1(self._f_bmap.read()).hexdigest()
225 self._f_bmap.seek(self._sha1_pos)
226 self._f_bmap.write("%s" % sha1)
227
228 def _calculate_sha1(self, first, last):
229 """ A helper function which calculates SHA1 checksum for the range of
230 blocks of the image file: from block 'first' to block 'last'. """
231
232 start = first * self.block_size
233 end = (last + 1) * self.block_size
234
235 self._f_image.seek(start)
236 hash_obj = hashlib.new("sha1")
237
238 chunk_size = 1024*1024
239 to_read = end - start
240 read = 0
241
242 while read < to_read:
243 if read + chunk_size > to_read:
244 chunk_size = to_read - read
245 chunk = self._f_image.read(chunk_size)
246 hash_obj.update(chunk)
247 read += chunk_size
248
249 return hash_obj.hexdigest()
250
251 def generate(self, include_checksums = True):
252 """ Generate bmap for the image file. If 'include_checksums' is 'True',
253 also generate SHA1 checksums for block ranges. """
254
255 # Save image file position in order to restore it at the end
256 image_pos = self._f_image.tell()
257
258 self._bmap_file_start()
259
260 # Generate the block map and write it to the XML block map
261 # file as we go.
262 self.mapped_cnt = 0
263 for first, last in self.fiemap.get_mapped_ranges(0, self.blocks_cnt):
264 self.mapped_cnt += last - first + 1
265 if include_checksums:
266 sha1 = self._calculate_sha1(first, last)
267 sha1 = " sha1=\"%s\"" % sha1
268 else:
269 sha1 = ""
270
271 if first != last:
272 self._f_bmap.write(" <Range%s> %s-%s </Range>\n" \
273 % (sha1, first, last))
274 else:
275 self._f_bmap.write(" <Range%s> %s </Range>\n" \
276 % (sha1, first))
277
278 self.mapped_size = self.mapped_cnt * self.block_size
279 self.mapped_size_human = human_size(self.mapped_size)
280 self.mapped_percent = (self.mapped_cnt * 100.0) / self.blocks_cnt
281
282 self._bmap_file_end()
283
284 try:
285 self._f_bmap.flush()
286 except IOError as err:
287 raise Error("cannot flush the bmap file '%s': %s" \
288 % (self._bmap_path, err))
289
290 self._f_image.seek(image_pos)
291
292 def __del__(self):
293 """ The class destructor which closes the opened files. """
294
295 if self._f_image_needs_close:
296 self._f_image.close()
297 if self._f_bmap_needs_close:
298 self._f_bmap.close()