summaryrefslogtreecommitdiffstats
path: root/scripts/lib
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/lib')
-rw-r--r--scripts/lib/wic/filemap.py309
1 files changed, 49 insertions, 260 deletions
diff --git a/scripts/lib/wic/filemap.py b/scripts/lib/wic/filemap.py
index 080668e7c2..34523c02b6 100644
--- a/scripts/lib/wic/filemap.py
+++ b/scripts/lib/wic/filemap.py
@@ -54,24 +54,46 @@ class Error(Exception):
54 """A class for all the other exceptions raised by this module.""" 54 """A class for all the other exceptions raised by this module."""
55 pass 55 pass
56 56
57# Below goes the FIEMAP ioctl implementation, which is not very readable
58# because it deals with the rather complex FIEMAP ioctl. To understand the
59# code, you need to know the FIEMAP interface, which is documented in the
60# "Documentation/filesystems/fiemap.txt" file in the Linux kernel sources.
61
62# Format string for 'struct fiemap'
63_FIEMAP_FORMAT = "=QQLLLL"
64# sizeof(struct fiemap)
65_FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT)
66# Format string for 'struct fiemap_extent'
67_FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL"
68# sizeof(struct fiemap_extent)
69_FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT)
70# The FIEMAP ioctl number
71_FIEMAP_IOCTL = 0xC020660B
72# This FIEMAP ioctl flag which instructs the kernel to sync the file before
73# reading the block map
74_FIEMAP_FLAG_SYNC = 0x00000001
75# Size of the buffer for 'struct fiemap_extent' elements which will be used
76# when invoking the FIEMAP ioctl. The larger is the buffer, the less times the
77# FIEMAP ioctl will be invoked.
78_FIEMAP_BUFFER_SIZE = 256 * 1024
57 79
58class _FilemapBase(object): 80class FilemapFiemap:
59 """ 81 """
60 This is a base class for a couple of other classes in this module. This 82 This class provides API to the FIEMAP ioctl. Namely, it allows to iterate
61 class simply performs the common parts of the initialization process: opens 83 over all mapped blocks and over all holes.
62 the image file, gets its size, etc. The 'log' parameter is the logger object 84
63 to use for printing messages. 85 This class synchronizes the image file every time it invokes the FIEMAP
86 ioctl in order to work-around early FIEMAP implementation kernel bugs.
64 """ 87 """
65 88
66 def __init__(self, image, log=None): 89 def __init__(self, image):
67 """ 90 """
68 Initialize a class instance. The 'image' argument is full path to the 91 Initialize a class instance. The 'image' argument is full the file
69 file or file object to operate on. 92 object to operate on.
70 """ 93 """
94 self._log = logging.getLogger(__name__)
71 95
72 self._log = log 96 self._log.debug("FilemapFiemap: initializing")
73 if self._log is None:
74 self._log = logging.getLogger(__name__)
75 97
76 self._f_image_needs_close = False 98 self._f_image_needs_close = False
77 99
@@ -113,240 +135,6 @@ class _FilemapBase(object):
113 self._log.debug("block size %d, blocks count %d, image size %d" 135 self._log.debug("block size %d, blocks count %d, image size %d"
114 % (self.block_size, self.blocks_cnt, self.image_size)) 136 % (self.block_size, self.blocks_cnt, self.image_size))
115 137
116 def __del__(self):
117 """The class destructor which just closes the image file."""
118 if self._f_image_needs_close:
119 self._f_image.close()
120
121 def _open_image_file(self):
122 """Open the image file."""
123 try:
124 self._f_image = open(self._image_path, 'rb')
125 except IOError as err:
126 raise Error("cannot open image file '%s': %s"
127 % (self._image_path, err))
128
129 self._f_image_needs_close = True
130
131 def block_is_mapped(self, block): # pylint: disable=W0613,R0201
132 """
133 This method has has to be implemented by child classes. It returns
134 'True' if block number 'block' of the image file is mapped and 'False'
135 otherwise.
136 """
137
138 raise Error("the method is not implemented")
139
140 def block_is_unmapped(self, block): # pylint: disable=W0613,R0201
141 """
142 This method has has to be implemented by child classes. It returns
143 'True' if block number 'block' of the image file is not mapped (hole)
144 and 'False' otherwise.
145 """
146
147 raise Error("the method is not implemented")
148
149 def get_mapped_ranges(self, start, count): # pylint: disable=W0613,R0201
150 """
151 This method has has to be implemented by child classes. This is a
152 generator which yields ranges of mapped blocks in the file. The ranges
153 are tuples of 2 elements: [first, last], where 'first' is the first
154 mapped block and 'last' is the last mapped block.
155
156 The ranges are yielded for the area of the file of size 'count' blocks,
157 starting from block 'start'.
158 """
159
160 raise Error("the method is not implemented")
161
162 def get_unmapped_ranges(self, start, count): # pylint: disable=W0613,R0201
163 """
164 This method has has to be implemented by child classes. Just like
165 'get_mapped_ranges()', but yields unmapped block ranges instead
166 (holes).
167 """
168
169 raise Error("the method is not implemented")
170
171
172# The 'SEEK_HOLE' and 'SEEK_DATA' options of the file seek system call
173_SEEK_DATA = 3
174_SEEK_HOLE = 4
175
176def _lseek(file_obj, offset, whence):
177 """This is a helper function which invokes 'os.lseek' for file object
178 'file_obj' and with specified 'offset' and 'whence'. The 'whence'
179 argument is supposed to be either '_SEEK_DATA' or '_SEEK_HOLE'. When
180 there is no more data or hole starting from 'offset', this function
181 returns '-1'. Otherwise the data or hole position is returned."""
182
183 try:
184 return os.lseek(file_obj.fileno(), offset, whence)
185 except OSError as err:
186 # The 'lseek' system call returns the ENXIO if there is no data or
187 # hole starting from the specified offset.
188 if err.errno == os.errno.ENXIO:
189 return -1
190 elif err.errno == os.errno.EINVAL:
191 raise ErrorNotSupp("the kernel or file-system does not support "
192 "\"SEEK_HOLE\" and \"SEEK_DATA\"")
193 else:
194 raise
195
196class FilemapSeek(_FilemapBase):
197 """
198 This class uses the 'SEEK_HOLE' and 'SEEK_DATA' to find file block mapping.
199 Unfortunately, the current implementation requires the caller to have write
200 access to the image file.
201 """
202
203 def __init__(self, image, log=None):
204 """Refer the '_FilemapBase' class for the documentation."""
205
206 # Call the base class constructor first
207 _FilemapBase.__init__(self, image, log)
208 self._log.debug("FilemapSeek: initializing")
209
210 self._probe_seek_hole()
211
212 def _probe_seek_hole(self):
213 """
214 Check whether the system implements 'SEEK_HOLE' and 'SEEK_DATA'.
215 Unfortunately, there seems to be no clean way for detecting this,
216 because often the system just fakes them by just assuming that all
217 files are fully mapped, so 'SEEK_HOLE' always returns EOF and
218 'SEEK_DATA' always returns the requested offset.
219
220 I could not invent a better way of detecting the fake 'SEEK_HOLE'
221 implementation than just to create a temporary file in the same
222 directory where the image file resides. It would be nice to change this
223 to something better.
224 """
225
226 directory = os.path.dirname(self._image_path)
227
228 try:
229 tmp_obj = tempfile.TemporaryFile("w+", dir=directory)
230 except IOError as err:
231 raise ErrorNotSupp("cannot create a temporary in \"%s\": %s"
232 % (directory, err))
233
234 try:
235 os.ftruncate(tmp_obj.fileno(), self.block_size)
236 except OSError as err:
237 raise ErrorNotSupp("cannot truncate temporary file in \"%s\": %s"
238 % (directory, err))
239
240 offs = _lseek(tmp_obj, 0, _SEEK_HOLE)
241 if offs != 0:
242 # We are dealing with the stub 'SEEK_HOLE' implementation which
243 # always returns EOF.
244 self._log.debug("lseek(0, SEEK_HOLE) returned %d" % offs)
245 raise ErrorNotSupp("the file-system does not support "
246 "\"SEEK_HOLE\" and \"SEEK_DATA\" but only "
247 "provides a stub implementation")
248
249 tmp_obj.close()
250
251 def block_is_mapped(self, block):
252 """Refer the '_FilemapBase' class for the documentation."""
253 offs = _lseek(self._f_image, block * self.block_size, _SEEK_DATA)
254 if offs == -1:
255 result = False
256 else:
257 result = (offs // self.block_size == block)
258
259 self._log.debug("FilemapSeek: block_is_mapped(%d) returns %s"
260 % (block, result))
261 return result
262
263 def block_is_unmapped(self, block):
264 """Refer the '_FilemapBase' class for the documentation."""
265 return not self.block_is_mapped(block)
266
267 def _get_ranges(self, start, count, whence1, whence2):
268 """
269 This function implements 'get_mapped_ranges()' and
270 'get_unmapped_ranges()' depending on what is passed in the 'whence1'
271 and 'whence2' arguments.
272 """
273
274 assert whence1 != whence2
275 end = start * self.block_size
276 limit = end + count * self.block_size
277
278 while True:
279 start = _lseek(self._f_image, end, whence1)
280 if start == -1 or start >= limit or start == self.image_size:
281 break
282
283 end = _lseek(self._f_image, start, whence2)
284 if end == -1 or end == self.image_size:
285 end = self.blocks_cnt * self.block_size
286 if end > limit:
287 end = limit
288
289 start_blk = start // self.block_size
290 end_blk = end // self.block_size - 1
291 self._log.debug("FilemapSeek: yielding range (%d, %d)"
292 % (start_blk, end_blk))
293 yield (start_blk, end_blk)
294
295 def get_mapped_ranges(self, start, count):
296 """Refer the '_FilemapBase' class for the documentation."""
297 self._log.debug("FilemapSeek: get_mapped_ranges(%d, %d(%d))"
298 % (start, count, start + count - 1))
299 return self._get_ranges(start, count, _SEEK_DATA, _SEEK_HOLE)
300
301 def get_unmapped_ranges(self, start, count):
302 """Refer the '_FilemapBase' class for the documentation."""
303 self._log.debug("FilemapSeek: get_unmapped_ranges(%d, %d(%d))"
304 % (start, count, start + count - 1))
305 return self._get_ranges(start, count, _SEEK_HOLE, _SEEK_DATA)
306
307
308# Below goes the FIEMAP ioctl implementation, which is not very readable
309# because it deals with the rather complex FIEMAP ioctl. To understand the
310# code, you need to know the FIEMAP interface, which is documented in the
311# "Documentation/filesystems/fiemap.txt" file in the Linux kernel sources.
312
313# Format string for 'struct fiemap'
314_FIEMAP_FORMAT = "=QQLLLL"
315# sizeof(struct fiemap)
316_FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT)
317# Format string for 'struct fiemap_extent'
318_FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL"
319# sizeof(struct fiemap_extent)
320_FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT)
321# The FIEMAP ioctl number
322_FIEMAP_IOCTL = 0xC020660B
323# This FIEMAP ioctl flag which instructs the kernel to sync the file before
324# reading the block map
325_FIEMAP_FLAG_SYNC = 0x00000001
326# Size of the buffer for 'struct fiemap_extent' elements which will be used
327# when invoking the FIEMAP ioctl. The larger is the buffer, the less times the
328# FIEMAP ioctl will be invoked.
329_FIEMAP_BUFFER_SIZE = 256 * 1024
330
331class FilemapFiemap(_FilemapBase):
332 """
333 This class provides API to the FIEMAP ioctl. Namely, it allows to iterate
334 over all mapped blocks and over all holes.
335
336 This class synchronizes the image file every time it invokes the FIEMAP
337 ioctl in order to work-around early FIEMAP implementation kernel bugs.
338 """
339
340 def __init__(self, image, log=None):
341 """
342 Initialize a class instance. The 'image' argument is full the file
343 object to operate on.
344 """
345
346 # Call the base class constructor first
347 _FilemapBase.__init__(self, image, log)
348 self._log.debug("FilemapFiemap: initializing")
349
350 self._buf_size = _FIEMAP_BUFFER_SIZE 138 self._buf_size = _FIEMAP_BUFFER_SIZE
351 139
352 # Calculate how many 'struct fiemap_extent' elements fit the buffer 140 # Calculate how many 'struct fiemap_extent' elements fit the buffer
@@ -362,6 +150,21 @@ class FilemapFiemap(_FilemapBase):
362 # Check if the FIEMAP ioctl is supported 150 # Check if the FIEMAP ioctl is supported
363 self.block_is_mapped(0) 151 self.block_is_mapped(0)
364 152
153 def __del__(self):
154 """The class destructor which just closes the image file."""
155 if self._f_image_needs_close:
156 self._f_image.close()
157
158 def _open_image_file(self):
159 """Open the image file."""
160 try:
161 self._f_image = open(self._image_path, 'rb')
162 except IOError as err:
163 raise Error("cannot open image file '%s': %s"
164 % (self._image_path, err))
165
166 self._f_image_needs_close = True
167
365 def _invoke_fiemap(self, block, count): 168 def _invoke_fiemap(self, block, count):
366 """ 169 """
367 Invoke the FIEMAP ioctl for 'count' blocks of the file starting from 170 Invoke the FIEMAP ioctl for 'count' blocks of the file starting from
@@ -515,24 +318,10 @@ class FilemapFiemap(_FilemapBase):
515 % (hole_first, start + count - 1)) 318 % (hole_first, start + count - 1))
516 yield (hole_first, start + count - 1) 319 yield (hole_first, start + count - 1)
517 320
518def filemap(image, log=None):
519 """
520 Create and return an instance of a Filemap class - 'FilemapFiemap' or
521 'FilemapSeek', depending on what the system we run on supports. If the
522 FIEMAP ioctl is supported, an instance of the 'FilemapFiemap' class is
523 returned. Otherwise, if 'SEEK_HOLE' is supported an instance of the
524 'FilemapSeek' class is returned. If none of these are supported, the
525 function generates an 'Error' type exception.
526 """
527
528 try:
529 return FilemapFiemap(image, log)
530 except ErrorNotSupp:
531 return FilemapSeek(image, log)
532 321
533def sparse_copy(src_fname, dst_fname, offset=0, skip=0): 322def sparse_copy(src_fname, dst_fname, offset=0, skip=0):
534 """Efficiently copy sparse file to or into another file.""" 323 """Efficiently copy sparse file to or into another file."""
535 fmap = filemap(src_fname) 324 fmap = FilemapFiemap(src_fname)
536 try: 325 try:
537 dst_file = open(dst_fname, 'r+b') 326 dst_file = open(dst_fname, 'r+b')
538 except IOError: 327 except IOError: