summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorEd Bartosh <ed.bartosh@linux.intel.com>2017-04-03 18:31:00 +0300
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-04-05 23:22:13 +0100
commitb5bc885ba703533b52ad092d3ee9a3090c00e499 (patch)
treeeeea16745861558b2f7927b0de814e67c4ba700e
parent86eaa6d83f4c8d9009a483578e0abd3add7e562a (diff)
downloadpoky-b5bc885ba703533b52ad092d3ee9a3090c00e499.tar.gz
Revert "filemap: remove FilemapSeek class"
FIEMAP API is not supported by tmpfs file system, but SEEK_HOLE/SEEK_DATA is supported. Returned back FilemapSeek class that implements support of SEEK_HOLE/SEEK_DATA API to make sparse_copy API working on tmpfs again. This reverts commit 6b80c13f7a82a312a3b981de5a56c66466ba1fac. (From OE-Core rev: e75bd6a7dd5c1b4bad039c35cf4a2ffc2f77c60a) Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--scripts/lib/wic/filemap.py309
1 files changed, 260 insertions, 49 deletions
diff --git a/scripts/lib/wic/filemap.py b/scripts/lib/wic/filemap.py
index 34523c02b6..080668e7c2 100644
--- a/scripts/lib/wic/filemap.py
+++ b/scripts/lib/wic/filemap.py
@@ -54,46 +54,24 @@ class Error(Exception):
54 """A class for all the other exceptions raised by this module.""" 54 """A class for all the other exceptions raised by this module."""
55 pass 55 pass
56 56
57# Below goes the FIEMAP ioctl implementation, which is not very readable
58# because it deals with the rather complex FIEMAP ioctl. To understand the
59# code, you need to know the FIEMAP interface, which is documented in the
60# "Documentation/filesystems/fiemap.txt" file in the Linux kernel sources.
61
62# Format string for 'struct fiemap'
63_FIEMAP_FORMAT = "=QQLLLL"
64# sizeof(struct fiemap)
65_FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT)
66# Format string for 'struct fiemap_extent'
67_FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL"
68# sizeof(struct fiemap_extent)
69_FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT)
70# The FIEMAP ioctl number
71_FIEMAP_IOCTL = 0xC020660B
72# This FIEMAP ioctl flag which instructs the kernel to sync the file before
73# reading the block map
74_FIEMAP_FLAG_SYNC = 0x00000001
75# Size of the buffer for 'struct fiemap_extent' elements which will be used
76# when invoking the FIEMAP ioctl. The larger is the buffer, the less times the
77# FIEMAP ioctl will be invoked.
78_FIEMAP_BUFFER_SIZE = 256 * 1024
79 57
80class FilemapFiemap: 58class _FilemapBase(object):
81 """ 59 """
82 This class provides API to the FIEMAP ioctl. Namely, it allows to iterate 60 This is a base class for a couple of other classes in this module. This
83 over all mapped blocks and over all holes. 61 class simply performs the common parts of the initialization process: opens
84 62 the image file, gets its size, etc. The 'log' parameter is the logger object
85 This class synchronizes the image file every time it invokes the FIEMAP 63 to use for printing messages.
86 ioctl in order to work-around early FIEMAP implementation kernel bugs.
87 """ 64 """
88 65
89 def __init__(self, image): 66 def __init__(self, image, log=None):
90 """ 67 """
91 Initialize a class instance. The 'image' argument is full the file 68 Initialize a class instance. The 'image' argument is full path to the
92 object to operate on. 69 file or file object to operate on.
93 """ 70 """
94 self._log = logging.getLogger(__name__)
95 71
96 self._log.debug("FilemapFiemap: initializing") 72 self._log = log
73 if self._log is None:
74 self._log = logging.getLogger(__name__)
97 75
98 self._f_image_needs_close = False 76 self._f_image_needs_close = False
99 77
@@ -135,21 +113,6 @@ class FilemapFiemap:
135 self._log.debug("block size %d, blocks count %d, image size %d" 113 self._log.debug("block size %d, blocks count %d, image size %d"
136 % (self.block_size, self.blocks_cnt, self.image_size)) 114 % (self.block_size, self.blocks_cnt, self.image_size))
137 115
138 self._buf_size = _FIEMAP_BUFFER_SIZE
139
140 # Calculate how many 'struct fiemap_extent' elements fit the buffer
141 self._buf_size -= _FIEMAP_SIZE
142 self._fiemap_extent_cnt = self._buf_size // _FIEMAP_EXTENT_SIZE
143 assert self._fiemap_extent_cnt > 0
144 self._buf_size = self._fiemap_extent_cnt * _FIEMAP_EXTENT_SIZE
145 self._buf_size += _FIEMAP_SIZE
146
147 # Allocate a mutable buffer for the FIEMAP ioctl
148 self._buf = array.array('B', [0] * self._buf_size)
149
150 # Check if the FIEMAP ioctl is supported
151 self.block_is_mapped(0)
152
153 def __del__(self): 116 def __del__(self):
154 """The class destructor which just closes the image file.""" 117 """The class destructor which just closes the image file."""
155 if self._f_image_needs_close: 118 if self._f_image_needs_close:
@@ -165,6 +128,240 @@ class FilemapFiemap:
165 128
166 self._f_image_needs_close = True 129 self._f_image_needs_close = True
167 130
131 def block_is_mapped(self, block): # pylint: disable=W0613,R0201
132 """
133 This method has has to be implemented by child classes. It returns
134 'True' if block number 'block' of the image file is mapped and 'False'
135 otherwise.
136 """
137
138 raise Error("the method is not implemented")
139
140 def block_is_unmapped(self, block): # pylint: disable=W0613,R0201
141 """
142 This method has has to be implemented by child classes. It returns
143 'True' if block number 'block' of the image file is not mapped (hole)
144 and 'False' otherwise.
145 """
146
147 raise Error("the method is not implemented")
148
149 def get_mapped_ranges(self, start, count): # pylint: disable=W0613,R0201
150 """
151 This method has has to be implemented by child classes. This is a
152 generator which yields ranges of mapped blocks in the file. The ranges
153 are tuples of 2 elements: [first, last], where 'first' is the first
154 mapped block and 'last' is the last mapped block.
155
156 The ranges are yielded for the area of the file of size 'count' blocks,
157 starting from block 'start'.
158 """
159
160 raise Error("the method is not implemented")
161
162 def get_unmapped_ranges(self, start, count): # pylint: disable=W0613,R0201
163 """
164 This method has has to be implemented by child classes. Just like
165 'get_mapped_ranges()', but yields unmapped block ranges instead
166 (holes).
167 """
168
169 raise Error("the method is not implemented")
170
171
172# The 'SEEK_HOLE' and 'SEEK_DATA' options of the file seek system call
173_SEEK_DATA = 3
174_SEEK_HOLE = 4
175
176def _lseek(file_obj, offset, whence):
177 """This is a helper function which invokes 'os.lseek' for file object
178 'file_obj' and with specified 'offset' and 'whence'. The 'whence'
179 argument is supposed to be either '_SEEK_DATA' or '_SEEK_HOLE'. When
180 there is no more data or hole starting from 'offset', this function
181 returns '-1'. Otherwise the data or hole position is returned."""
182
183 try:
184 return os.lseek(file_obj.fileno(), offset, whence)
185 except OSError as err:
186 # The 'lseek' system call returns the ENXIO if there is no data or
187 # hole starting from the specified offset.
188 if err.errno == os.errno.ENXIO:
189 return -1
190 elif err.errno == os.errno.EINVAL:
191 raise ErrorNotSupp("the kernel or file-system does not support "
192 "\"SEEK_HOLE\" and \"SEEK_DATA\"")
193 else:
194 raise
195
196class FilemapSeek(_FilemapBase):
197 """
198 This class uses the 'SEEK_HOLE' and 'SEEK_DATA' to find file block mapping.
199 Unfortunately, the current implementation requires the caller to have write
200 access to the image file.
201 """
202
203 def __init__(self, image, log=None):
204 """Refer the '_FilemapBase' class for the documentation."""
205
206 # Call the base class constructor first
207 _FilemapBase.__init__(self, image, log)
208 self._log.debug("FilemapSeek: initializing")
209
210 self._probe_seek_hole()
211
212 def _probe_seek_hole(self):
213 """
214 Check whether the system implements 'SEEK_HOLE' and 'SEEK_DATA'.
215 Unfortunately, there seems to be no clean way for detecting this,
216 because often the system just fakes them by just assuming that all
217 files are fully mapped, so 'SEEK_HOLE' always returns EOF and
218 'SEEK_DATA' always returns the requested offset.
219
220 I could not invent a better way of detecting the fake 'SEEK_HOLE'
221 implementation than just to create a temporary file in the same
222 directory where the image file resides. It would be nice to change this
223 to something better.
224 """
225
226 directory = os.path.dirname(self._image_path)
227
228 try:
229 tmp_obj = tempfile.TemporaryFile("w+", dir=directory)
230 except IOError as err:
231 raise ErrorNotSupp("cannot create a temporary in \"%s\": %s"
232 % (directory, err))
233
234 try:
235 os.ftruncate(tmp_obj.fileno(), self.block_size)
236 except OSError as err:
237 raise ErrorNotSupp("cannot truncate temporary file in \"%s\": %s"
238 % (directory, err))
239
240 offs = _lseek(tmp_obj, 0, _SEEK_HOLE)
241 if offs != 0:
242 # We are dealing with the stub 'SEEK_HOLE' implementation which
243 # always returns EOF.
244 self._log.debug("lseek(0, SEEK_HOLE) returned %d" % offs)
245 raise ErrorNotSupp("the file-system does not support "
246 "\"SEEK_HOLE\" and \"SEEK_DATA\" but only "
247 "provides a stub implementation")
248
249 tmp_obj.close()
250
251 def block_is_mapped(self, block):
252 """Refer the '_FilemapBase' class for the documentation."""
253 offs = _lseek(self._f_image, block * self.block_size, _SEEK_DATA)
254 if offs == -1:
255 result = False
256 else:
257 result = (offs // self.block_size == block)
258
259 self._log.debug("FilemapSeek: block_is_mapped(%d) returns %s"
260 % (block, result))
261 return result
262
263 def block_is_unmapped(self, block):
264 """Refer the '_FilemapBase' class for the documentation."""
265 return not self.block_is_mapped(block)
266
267 def _get_ranges(self, start, count, whence1, whence2):
268 """
269 This function implements 'get_mapped_ranges()' and
270 'get_unmapped_ranges()' depending on what is passed in the 'whence1'
271 and 'whence2' arguments.
272 """
273
274 assert whence1 != whence2
275 end = start * self.block_size
276 limit = end + count * self.block_size
277
278 while True:
279 start = _lseek(self._f_image, end, whence1)
280 if start == -1 or start >= limit or start == self.image_size:
281 break
282
283 end = _lseek(self._f_image, start, whence2)
284 if end == -1 or end == self.image_size:
285 end = self.blocks_cnt * self.block_size
286 if end > limit:
287 end = limit
288
289 start_blk = start // self.block_size
290 end_blk = end // self.block_size - 1
291 self._log.debug("FilemapSeek: yielding range (%d, %d)"
292 % (start_blk, end_blk))
293 yield (start_blk, end_blk)
294
295 def get_mapped_ranges(self, start, count):
296 """Refer the '_FilemapBase' class for the documentation."""
297 self._log.debug("FilemapSeek: get_mapped_ranges(%d, %d(%d))"
298 % (start, count, start + count - 1))
299 return self._get_ranges(start, count, _SEEK_DATA, _SEEK_HOLE)
300
301 def get_unmapped_ranges(self, start, count):
302 """Refer the '_FilemapBase' class for the documentation."""
303 self._log.debug("FilemapSeek: get_unmapped_ranges(%d, %d(%d))"
304 % (start, count, start + count - 1))
305 return self._get_ranges(start, count, _SEEK_HOLE, _SEEK_DATA)
306
307
308# Below goes the FIEMAP ioctl implementation, which is not very readable
309# because it deals with the rather complex FIEMAP ioctl. To understand the
310# code, you need to know the FIEMAP interface, which is documented in the
311# "Documentation/filesystems/fiemap.txt" file in the Linux kernel sources.
312
313# Format string for 'struct fiemap'
314_FIEMAP_FORMAT = "=QQLLLL"
315# sizeof(struct fiemap)
316_FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT)
317# Format string for 'struct fiemap_extent'
318_FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL"
319# sizeof(struct fiemap_extent)
320_FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT)
321# The FIEMAP ioctl number
322_FIEMAP_IOCTL = 0xC020660B
323# This FIEMAP ioctl flag which instructs the kernel to sync the file before
324# reading the block map
325_FIEMAP_FLAG_SYNC = 0x00000001
326# Size of the buffer for 'struct fiemap_extent' elements which will be used
327# when invoking the FIEMAP ioctl. The larger is the buffer, the less times the
328# FIEMAP ioctl will be invoked.
329_FIEMAP_BUFFER_SIZE = 256 * 1024
330
331class FilemapFiemap(_FilemapBase):
332 """
333 This class provides API to the FIEMAP ioctl. Namely, it allows to iterate
334 over all mapped blocks and over all holes.
335
336 This class synchronizes the image file every time it invokes the FIEMAP
337 ioctl in order to work-around early FIEMAP implementation kernel bugs.
338 """
339
340 def __init__(self, image, log=None):
341 """
342 Initialize a class instance. The 'image' argument is full the file
343 object to operate on.
344 """
345
346 # Call the base class constructor first
347 _FilemapBase.__init__(self, image, log)
348 self._log.debug("FilemapFiemap: initializing")
349
350 self._buf_size = _FIEMAP_BUFFER_SIZE
351
352 # Calculate how many 'struct fiemap_extent' elements fit the buffer
353 self._buf_size -= _FIEMAP_SIZE
354 self._fiemap_extent_cnt = self._buf_size // _FIEMAP_EXTENT_SIZE
355 assert self._fiemap_extent_cnt > 0
356 self._buf_size = self._fiemap_extent_cnt * _FIEMAP_EXTENT_SIZE
357 self._buf_size += _FIEMAP_SIZE
358
359 # Allocate a mutable buffer for the FIEMAP ioctl
360 self._buf = array.array('B', [0] * self._buf_size)
361
362 # Check if the FIEMAP ioctl is supported
363 self.block_is_mapped(0)
364
168 def _invoke_fiemap(self, block, count): 365 def _invoke_fiemap(self, block, count):
169 """ 366 """
170 Invoke the FIEMAP ioctl for 'count' blocks of the file starting from 367 Invoke the FIEMAP ioctl for 'count' blocks of the file starting from
@@ -318,10 +515,24 @@ class FilemapFiemap:
318 % (hole_first, start + count - 1)) 515 % (hole_first, start + count - 1))
319 yield (hole_first, start + count - 1) 516 yield (hole_first, start + count - 1)
320 517
518def filemap(image, log=None):
519 """
520 Create and return an instance of a Filemap class - 'FilemapFiemap' or
521 'FilemapSeek', depending on what the system we run on supports. If the
522 FIEMAP ioctl is supported, an instance of the 'FilemapFiemap' class is
523 returned. Otherwise, if 'SEEK_HOLE' is supported an instance of the
524 'FilemapSeek' class is returned. If none of these are supported, the
525 function generates an 'Error' type exception.
526 """
527
528 try:
529 return FilemapFiemap(image, log)
530 except ErrorNotSupp:
531 return FilemapSeek(image, log)
321 532
322def sparse_copy(src_fname, dst_fname, offset=0, skip=0): 533def sparse_copy(src_fname, dst_fname, offset=0, skip=0):
323 """Efficiently copy sparse file to or into another file.""" 534 """Efficiently copy sparse file to or into another file."""
324 fmap = FilemapFiemap(src_fname) 535 fmap = filemap(src_fname)
325 try: 536 try:
326 dst_file = open(dst_fname, 'r+b') 537 dst_file = open(dst_fname, 'r+b')
327 except IOError: 538 except IOError: