1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
|
""" This module implements the block map (bmap) creation functionality and
provides the corresponding API in form of the 'BmapCreate' class.
The idea is that while images files may generally be very large (e.g., 4GiB),
they may nevertheless contain only little real data, e.g., 512MiB. This data
are files, directories, file-system meta-data, partition table, etc. When
copying the image to the target device, you do not have to copy all the 4GiB of
data, you can copy only 512MiB of it, which is 4 times less, so copying should
presumably be 4 times faster.
The block map file is an XML file which contains a list of blocks which have to
be copied to the target device. The other blocks are not used and there is no
need to copy them. The XML file also contains some additional information like
block size, image size, count of mapped blocks, etc. There are also many
commentaries, so it is human-readable.
The image has to be a sparse file. Generally, this means that when you generate
this image file, you should start with a huge sparse file which contains a
single hole spanning the entire file. Then you should partition it, write all
the data (probably by means of loop-back mounting the image or parts of it),
etc. The end result should be a sparse file where mapped areas represent useful
parts of the image and holes represent useless parts of the image, which do not
have to be copied when copying the image to the target device.
This module uses the FIBMAP ioctl to detect holes. """
# Disable the following pylint recommendations:
# * Too many instance attributes - R0902
# * Too few public methods - R0903
# pylint: disable=R0902,R0903
import hashlib
from mic.utils.misc import human_size
from mic.utils import Fiemap
# The bmap format version we generate
SUPPORTED_BMAP_VERSION = "1.3"
_BMAP_START_TEMPLATE = \
"""<?xml version="1.0" ?>
<!-- This file contains the block map for an image file, which is basically
a list of useful (mapped) block numbers in the image file. In other words,
it lists only those blocks which contain data (boot sector, partition
table, file-system metadata, files, directories, extents, etc). These
blocks have to be copied to the target device. The other blocks do not
contain any useful data and do not have to be copied to the target
device.
The block map an optimization which allows to copy or flash the image to
the image quicker than copying of flashing the entire image. This is
because with bmap less data is copied: <MappedBlocksCount> blocks instead
of <BlocksCount> blocks.
Besides the machine-readable data, this file contains useful commentaries
which contain human-readable information like image size, percentage of
mapped data, etc.
The 'version' attribute is the block map file format version in the
'major.minor' format. The version major number is increased whenever an
incompatible block map format change is made. The minor number changes
in case of minor backward-compatible changes. -->
<bmap version="%s">
<!-- Image size in bytes: %s -->
<ImageSize> %u </ImageSize>
<!-- Size of a block in bytes -->
<BlockSize> %u </BlockSize>
<!-- Count of blocks in the image file -->
<BlocksCount> %u </BlocksCount>
"""
class Error(Exception):
""" A class for exceptions generated by this module. We currently support
only one type of exceptions, and we basically throw human-readable problem
description in case of errors. """
pass
class BmapCreate:
""" This class implements the bmap creation functionality. To generate a
bmap for an image (which is supposedly a sparse file), you should first
create an instance of 'BmapCreate' and provide:
* full path or a file-like object of the image to create bmap for
* full path or a file object to use for writing the results to
Then you should invoke the 'generate()' method of this class. It will use
the FIEMAP ioctl to generate the bmap. """
def _open_image_file(self):
""" Open the image file. """
try:
self._f_image = open(self._image_path, 'rb')
except IOError as err:
raise Error("cannot open image file '%s': %s" \
% (self._image_path, err))
self._f_image_needs_close = True
def _open_bmap_file(self):
""" Open the bmap file. """
try:
self._f_bmap = open(self._bmap_path, 'w+')
except IOError as err:
raise Error("cannot open bmap file '%s': %s" \
% (self._bmap_path, err))
self._f_bmap_needs_close = True
def __init__(self, image, bmap):
""" Initialize a class instance:
* image - full path or a file-like object of the image to create bmap
for
* bmap - full path or a file object to use for writing the resulting
bmap to """
self.image_size = None
self.image_size_human = None
self.block_size = None
self.blocks_cnt = None
self.mapped_cnt = None
self.mapped_size = None
self.mapped_size_human = None
self.mapped_percent = None
self._mapped_count_pos1 = None
self._mapped_count_pos2 = None
self._sha1_pos = None
self._f_image_needs_close = False
self._f_bmap_needs_close = False
if hasattr(image, "read"):
self._f_image = image
self._image_path = image.name
else:
self._image_path = image
self._open_image_file()
if hasattr(bmap, "read"):
self._f_bmap = bmap
self._bmap_path = bmap.name
else:
self._bmap_path = bmap
self._open_bmap_file()
self.fiemap = Fiemap.Fiemap(self._f_image)
self.image_size = self.fiemap.image_size
self.image_size_human = human_size(self.image_size)
if self.image_size == 0:
raise Error("cannot generate bmap for zero-sized image file '%s'" \
% self._image_path)
self.block_size = self.fiemap.block_size
self.blocks_cnt = self.fiemap.blocks_cnt
def _bmap_file_start(self):
""" A helper function which generates the starting contents of the
block map file: the header comment, image size, block size, etc. """
# We do not know the amount of mapped blocks at the moment, so just put
# whitespaces instead of real numbers. Assume the longest possible
# numbers.
mapped_count = ' ' * len(str(self.image_size))
mapped_size_human = ' ' * len(self.image_size_human)
xml = _BMAP_START_TEMPLATE \
% (SUPPORTED_BMAP_VERSION, self.image_size_human,
self.image_size, self.block_size, self.blocks_cnt)
xml += " <!-- Count of mapped blocks: "
self._f_bmap.write(xml)
self._mapped_count_pos1 = self._f_bmap.tell()
# Just put white-spaces instead of real information about mapped blocks
xml = "%s or %.1f -->\n" % (mapped_size_human, 100.0)
xml += " <MappedBlocksCount> "
self._f_bmap.write(xml)
self._mapped_count_pos2 = self._f_bmap.tell()
xml = "%s </MappedBlocksCount>\n\n" % mapped_count
# pylint: disable=C0301
xml += " <!-- The checksum of this bmap file. When it is calculated, the value of\n"
xml += " the SHA1 checksum has be zeoro (40 ASCII \"0\" symbols). -->\n"
xml += " <BmapFileSHA1> "
self._f_bmap.write(xml)
self._sha1_pos = self._f_bmap.tell()
xml = "0" * 40 + " </BmapFileSHA1>\n\n"
xml += " <!-- The block map which consists of elements which may either be a\n"
xml += " range of blocks or a single block. The 'sha1' attribute (if present)\n"
xml += " is the SHA1 checksum of this blocks range. -->\n"
xml += " <BlockMap>\n"
# pylint: enable=C0301
self._f_bmap.write(xml)
def _bmap_file_end(self):
""" A helper function which generates the final parts of the block map
file: the ending tags and the information about the amount of mapped
blocks. """
xml = " </BlockMap>\n"
xml += "</bmap>\n"
self._f_bmap.write(xml)
self._f_bmap.seek(self._mapped_count_pos1)
self._f_bmap.write("%s or %.1f%%" % \
(self.mapped_size_human, self.mapped_percent))
self._f_bmap.seek(self._mapped_count_pos2)
self._f_bmap.write("%u" % self.mapped_cnt)
self._f_bmap.seek(0)
sha1 = hashlib.sha1(self._f_bmap.read()).hexdigest()
self._f_bmap.seek(self._sha1_pos)
self._f_bmap.write("%s" % sha1)
def _calculate_sha1(self, first, last):
""" A helper function which calculates SHA1 checksum for the range of
blocks of the image file: from block 'first' to block 'last'. """
start = first * self.block_size
end = (last + 1) * self.block_size
self._f_image.seek(start)
hash_obj = hashlib.new("sha1")
chunk_size = 1024*1024
to_read = end - start
read = 0
while read < to_read:
if read + chunk_size > to_read:
chunk_size = to_read - read
chunk = self._f_image.read(chunk_size)
hash_obj.update(chunk)
read += chunk_size
return hash_obj.hexdigest()
def generate(self, include_checksums = True):
""" Generate bmap for the image file. If 'include_checksums' is 'True',
also generate SHA1 checksums for block ranges. """
# Save image file position in order to restore it at the end
image_pos = self._f_image.tell()
self._bmap_file_start()
# Generate the block map and write it to the XML block map
# file as we go.
self.mapped_cnt = 0
for first, last in self.fiemap.get_mapped_ranges(0, self.blocks_cnt):
self.mapped_cnt += last - first + 1
if include_checksums:
sha1 = self._calculate_sha1(first, last)
sha1 = " sha1=\"%s\"" % sha1
else:
sha1 = ""
if first != last:
self._f_bmap.write(" <Range%s> %s-%s </Range>\n" \
% (sha1, first, last))
else:
self._f_bmap.write(" <Range%s> %s </Range>\n" \
% (sha1, first))
self.mapped_size = self.mapped_cnt * self.block_size
self.mapped_size_human = human_size(self.mapped_size)
self.mapped_percent = (self.mapped_cnt * 100.0) / self.blocks_cnt
self._bmap_file_end()
try:
self._f_bmap.flush()
except IOError as err:
raise Error("cannot flush the bmap file '%s': %s" \
% (self._bmap_path, err))
self._f_image.seek(image_pos)
def __del__(self):
""" The class destructor which closes the opened files. """
if self._f_image_needs_close:
self._f_image.close()
if self._f_bmap_needs_close:
self._f_bmap.close()
|
