diff options
author | Ed Bartosh <ed.bartosh@linux.intel.com> | 2017-03-26 20:08:46 +0300 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2017-04-01 08:14:57 +0100 |
commit | 6b80c13f7a82a312a3b981de5a56c66466ba1fac (patch) | |
tree | eaa6074019745c74366fad7c6f6017666f4f7d83 /scripts/lib/wic/filemap.py | |
parent | fae19c345de67663f577b2fc9e93070ea976f211 (diff) | |
download | poky-6b80c13f7a82a312a3b981de5a56c66466ba1fac.tar.gz |
filemap: remove FilemapSeek class
FIEMAP API was added to Linux kernel 2.6.28 back in 2008
SEEK_HOLE and SEEK_DATA API was added much letter.
As FIEMAP is used by filemap module as a default API it's
safe to remove FileMpSeek class as it's never used.
[YOCTO #10618]
(From OE-Core rev: 44e9406ea6e3263d2fb95e9d534a21f74f318480)
Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/wic/filemap.py')
-rw-r--r-- | scripts/lib/wic/filemap.py | 309 |
1 files changed, 49 insertions, 260 deletions
diff --git a/scripts/lib/wic/filemap.py b/scripts/lib/wic/filemap.py index 080668e7c2..34523c02b6 100644 --- a/scripts/lib/wic/filemap.py +++ b/scripts/lib/wic/filemap.py | |||
@@ -54,24 +54,46 @@ class Error(Exception): | |||
54 | """A class for all the other exceptions raised by this module.""" | 54 | """A class for all the other exceptions raised by this module.""" |
55 | pass | 55 | pass |
56 | 56 | ||
57 | # Below goes the FIEMAP ioctl implementation, which is not very readable | ||
58 | # because it deals with the rather complex FIEMAP ioctl. To understand the | ||
59 | # code, you need to know the FIEMAP interface, which is documented in the | ||
60 | # "Documentation/filesystems/fiemap.txt" file in the Linux kernel sources. | ||
61 | |||
62 | # Format string for 'struct fiemap' | ||
63 | _FIEMAP_FORMAT = "=QQLLLL" | ||
64 | # sizeof(struct fiemap) | ||
65 | _FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT) | ||
66 | # Format string for 'struct fiemap_extent' | ||
67 | _FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL" | ||
68 | # sizeof(struct fiemap_extent) | ||
69 | _FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT) | ||
70 | # The FIEMAP ioctl number | ||
71 | _FIEMAP_IOCTL = 0xC020660B | ||
72 | # This FIEMAP ioctl flag which instructs the kernel to sync the file before | ||
73 | # reading the block map | ||
74 | _FIEMAP_FLAG_SYNC = 0x00000001 | ||
75 | # Size of the buffer for 'struct fiemap_extent' elements which will be used | ||
76 | # when invoking the FIEMAP ioctl. The larger is the buffer, the less times the | ||
77 | # FIEMAP ioctl will be invoked. | ||
78 | _FIEMAP_BUFFER_SIZE = 256 * 1024 | ||
57 | 79 | ||
58 | class _FilemapBase(object): | 80 | class FilemapFiemap: |
59 | """ | 81 | """ |
60 | This is a base class for a couple of other classes in this module. This | 82 | This class provides API to the FIEMAP ioctl. Namely, it allows to iterate |
61 | class simply performs the common parts of the initialization process: opens | 83 | over all mapped blocks and over all holes. |
62 | the image file, gets its size, etc. The 'log' parameter is the logger object | 84 | |
63 | to use for printing messages. | 85 | This class synchronizes the image file every time it invokes the FIEMAP |
86 | ioctl in order to work-around early FIEMAP implementation kernel bugs. | ||
64 | """ | 87 | """ |
65 | 88 | ||
66 | def __init__(self, image, log=None): | 89 | def __init__(self, image): |
67 | """ | 90 | """ |
68 | Initialize a class instance. The 'image' argument is full path to the | 91 | Initialize a class instance. The 'image' argument is full the file |
69 | file or file object to operate on. | 92 | object to operate on. |
70 | """ | 93 | """ |
94 | self._log = logging.getLogger(__name__) | ||
71 | 95 | ||
72 | self._log = log | 96 | self._log.debug("FilemapFiemap: initializing") |
73 | if self._log is None: | ||
74 | self._log = logging.getLogger(__name__) | ||
75 | 97 | ||
76 | self._f_image_needs_close = False | 98 | self._f_image_needs_close = False |
77 | 99 | ||
@@ -113,240 +135,6 @@ class _FilemapBase(object): | |||
113 | self._log.debug("block size %d, blocks count %d, image size %d" | 135 | self._log.debug("block size %d, blocks count %d, image size %d" |
114 | % (self.block_size, self.blocks_cnt, self.image_size)) | 136 | % (self.block_size, self.blocks_cnt, self.image_size)) |
115 | 137 | ||
116 | def __del__(self): | ||
117 | """The class destructor which just closes the image file.""" | ||
118 | if self._f_image_needs_close: | ||
119 | self._f_image.close() | ||
120 | |||
121 | def _open_image_file(self): | ||
122 | """Open the image file.""" | ||
123 | try: | ||
124 | self._f_image = open(self._image_path, 'rb') | ||
125 | except IOError as err: | ||
126 | raise Error("cannot open image file '%s': %s" | ||
127 | % (self._image_path, err)) | ||
128 | |||
129 | self._f_image_needs_close = True | ||
130 | |||
131 | def block_is_mapped(self, block): # pylint: disable=W0613,R0201 | ||
132 | """ | ||
133 | This method has has to be implemented by child classes. It returns | ||
134 | 'True' if block number 'block' of the image file is mapped and 'False' | ||
135 | otherwise. | ||
136 | """ | ||
137 | |||
138 | raise Error("the method is not implemented") | ||
139 | |||
140 | def block_is_unmapped(self, block): # pylint: disable=W0613,R0201 | ||
141 | """ | ||
142 | This method has has to be implemented by child classes. It returns | ||
143 | 'True' if block number 'block' of the image file is not mapped (hole) | ||
144 | and 'False' otherwise. | ||
145 | """ | ||
146 | |||
147 | raise Error("the method is not implemented") | ||
148 | |||
149 | def get_mapped_ranges(self, start, count): # pylint: disable=W0613,R0201 | ||
150 | """ | ||
151 | This method has has to be implemented by child classes. This is a | ||
152 | generator which yields ranges of mapped blocks in the file. The ranges | ||
153 | are tuples of 2 elements: [first, last], where 'first' is the first | ||
154 | mapped block and 'last' is the last mapped block. | ||
155 | |||
156 | The ranges are yielded for the area of the file of size 'count' blocks, | ||
157 | starting from block 'start'. | ||
158 | """ | ||
159 | |||
160 | raise Error("the method is not implemented") | ||
161 | |||
162 | def get_unmapped_ranges(self, start, count): # pylint: disable=W0613,R0201 | ||
163 | """ | ||
164 | This method has has to be implemented by child classes. Just like | ||
165 | 'get_mapped_ranges()', but yields unmapped block ranges instead | ||
166 | (holes). | ||
167 | """ | ||
168 | |||
169 | raise Error("the method is not implemented") | ||
170 | |||
171 | |||
172 | # The 'SEEK_HOLE' and 'SEEK_DATA' options of the file seek system call | ||
173 | _SEEK_DATA = 3 | ||
174 | _SEEK_HOLE = 4 | ||
175 | |||
176 | def _lseek(file_obj, offset, whence): | ||
177 | """This is a helper function which invokes 'os.lseek' for file object | ||
178 | 'file_obj' and with specified 'offset' and 'whence'. The 'whence' | ||
179 | argument is supposed to be either '_SEEK_DATA' or '_SEEK_HOLE'. When | ||
180 | there is no more data or hole starting from 'offset', this function | ||
181 | returns '-1'. Otherwise the data or hole position is returned.""" | ||
182 | |||
183 | try: | ||
184 | return os.lseek(file_obj.fileno(), offset, whence) | ||
185 | except OSError as err: | ||
186 | # The 'lseek' system call returns the ENXIO if there is no data or | ||
187 | # hole starting from the specified offset. | ||
188 | if err.errno == os.errno.ENXIO: | ||
189 | return -1 | ||
190 | elif err.errno == os.errno.EINVAL: | ||
191 | raise ErrorNotSupp("the kernel or file-system does not support " | ||
192 | "\"SEEK_HOLE\" and \"SEEK_DATA\"") | ||
193 | else: | ||
194 | raise | ||
195 | |||
196 | class FilemapSeek(_FilemapBase): | ||
197 | """ | ||
198 | This class uses the 'SEEK_HOLE' and 'SEEK_DATA' to find file block mapping. | ||
199 | Unfortunately, the current implementation requires the caller to have write | ||
200 | access to the image file. | ||
201 | """ | ||
202 | |||
203 | def __init__(self, image, log=None): | ||
204 | """Refer the '_FilemapBase' class for the documentation.""" | ||
205 | |||
206 | # Call the base class constructor first | ||
207 | _FilemapBase.__init__(self, image, log) | ||
208 | self._log.debug("FilemapSeek: initializing") | ||
209 | |||
210 | self._probe_seek_hole() | ||
211 | |||
212 | def _probe_seek_hole(self): | ||
213 | """ | ||
214 | Check whether the system implements 'SEEK_HOLE' and 'SEEK_DATA'. | ||
215 | Unfortunately, there seems to be no clean way for detecting this, | ||
216 | because often the system just fakes them by just assuming that all | ||
217 | files are fully mapped, so 'SEEK_HOLE' always returns EOF and | ||
218 | 'SEEK_DATA' always returns the requested offset. | ||
219 | |||
220 | I could not invent a better way of detecting the fake 'SEEK_HOLE' | ||
221 | implementation than just to create a temporary file in the same | ||
222 | directory where the image file resides. It would be nice to change this | ||
223 | to something better. | ||
224 | """ | ||
225 | |||
226 | directory = os.path.dirname(self._image_path) | ||
227 | |||
228 | try: | ||
229 | tmp_obj = tempfile.TemporaryFile("w+", dir=directory) | ||
230 | except IOError as err: | ||
231 | raise ErrorNotSupp("cannot create a temporary in \"%s\": %s" | ||
232 | % (directory, err)) | ||
233 | |||
234 | try: | ||
235 | os.ftruncate(tmp_obj.fileno(), self.block_size) | ||
236 | except OSError as err: | ||
237 | raise ErrorNotSupp("cannot truncate temporary file in \"%s\": %s" | ||
238 | % (directory, err)) | ||
239 | |||
240 | offs = _lseek(tmp_obj, 0, _SEEK_HOLE) | ||
241 | if offs != 0: | ||
242 | # We are dealing with the stub 'SEEK_HOLE' implementation which | ||
243 | # always returns EOF. | ||
244 | self._log.debug("lseek(0, SEEK_HOLE) returned %d" % offs) | ||
245 | raise ErrorNotSupp("the file-system does not support " | ||
246 | "\"SEEK_HOLE\" and \"SEEK_DATA\" but only " | ||
247 | "provides a stub implementation") | ||
248 | |||
249 | tmp_obj.close() | ||
250 | |||
251 | def block_is_mapped(self, block): | ||
252 | """Refer the '_FilemapBase' class for the documentation.""" | ||
253 | offs = _lseek(self._f_image, block * self.block_size, _SEEK_DATA) | ||
254 | if offs == -1: | ||
255 | result = False | ||
256 | else: | ||
257 | result = (offs // self.block_size == block) | ||
258 | |||
259 | self._log.debug("FilemapSeek: block_is_mapped(%d) returns %s" | ||
260 | % (block, result)) | ||
261 | return result | ||
262 | |||
263 | def block_is_unmapped(self, block): | ||
264 | """Refer the '_FilemapBase' class for the documentation.""" | ||
265 | return not self.block_is_mapped(block) | ||
266 | |||
267 | def _get_ranges(self, start, count, whence1, whence2): | ||
268 | """ | ||
269 | This function implements 'get_mapped_ranges()' and | ||
270 | 'get_unmapped_ranges()' depending on what is passed in the 'whence1' | ||
271 | and 'whence2' arguments. | ||
272 | """ | ||
273 | |||
274 | assert whence1 != whence2 | ||
275 | end = start * self.block_size | ||
276 | limit = end + count * self.block_size | ||
277 | |||
278 | while True: | ||
279 | start = _lseek(self._f_image, end, whence1) | ||
280 | if start == -1 or start >= limit or start == self.image_size: | ||
281 | break | ||
282 | |||
283 | end = _lseek(self._f_image, start, whence2) | ||
284 | if end == -1 or end == self.image_size: | ||
285 | end = self.blocks_cnt * self.block_size | ||
286 | if end > limit: | ||
287 | end = limit | ||
288 | |||
289 | start_blk = start // self.block_size | ||
290 | end_blk = end // self.block_size - 1 | ||
291 | self._log.debug("FilemapSeek: yielding range (%d, %d)" | ||
292 | % (start_blk, end_blk)) | ||
293 | yield (start_blk, end_blk) | ||
294 | |||
295 | def get_mapped_ranges(self, start, count): | ||
296 | """Refer the '_FilemapBase' class for the documentation.""" | ||
297 | self._log.debug("FilemapSeek: get_mapped_ranges(%d, %d(%d))" | ||
298 | % (start, count, start + count - 1)) | ||
299 | return self._get_ranges(start, count, _SEEK_DATA, _SEEK_HOLE) | ||
300 | |||
301 | def get_unmapped_ranges(self, start, count): | ||
302 | """Refer the '_FilemapBase' class for the documentation.""" | ||
303 | self._log.debug("FilemapSeek: get_unmapped_ranges(%d, %d(%d))" | ||
304 | % (start, count, start + count - 1)) | ||
305 | return self._get_ranges(start, count, _SEEK_HOLE, _SEEK_DATA) | ||
306 | |||
307 | |||
308 | # Below goes the FIEMAP ioctl implementation, which is not very readable | ||
309 | # because it deals with the rather complex FIEMAP ioctl. To understand the | ||
310 | # code, you need to know the FIEMAP interface, which is documented in the | ||
311 | # "Documentation/filesystems/fiemap.txt" file in the Linux kernel sources. | ||
312 | |||
313 | # Format string for 'struct fiemap' | ||
314 | _FIEMAP_FORMAT = "=QQLLLL" | ||
315 | # sizeof(struct fiemap) | ||
316 | _FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT) | ||
317 | # Format string for 'struct fiemap_extent' | ||
318 | _FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL" | ||
319 | # sizeof(struct fiemap_extent) | ||
320 | _FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT) | ||
321 | # The FIEMAP ioctl number | ||
322 | _FIEMAP_IOCTL = 0xC020660B | ||
323 | # This FIEMAP ioctl flag which instructs the kernel to sync the file before | ||
324 | # reading the block map | ||
325 | _FIEMAP_FLAG_SYNC = 0x00000001 | ||
326 | # Size of the buffer for 'struct fiemap_extent' elements which will be used | ||
327 | # when invoking the FIEMAP ioctl. The larger is the buffer, the less times the | ||
328 | # FIEMAP ioctl will be invoked. | ||
329 | _FIEMAP_BUFFER_SIZE = 256 * 1024 | ||
330 | |||
331 | class FilemapFiemap(_FilemapBase): | ||
332 | """ | ||
333 | This class provides API to the FIEMAP ioctl. Namely, it allows to iterate | ||
334 | over all mapped blocks and over all holes. | ||
335 | |||
336 | This class synchronizes the image file every time it invokes the FIEMAP | ||
337 | ioctl in order to work-around early FIEMAP implementation kernel bugs. | ||
338 | """ | ||
339 | |||
340 | def __init__(self, image, log=None): | ||
341 | """ | ||
342 | Initialize a class instance. The 'image' argument is full the file | ||
343 | object to operate on. | ||
344 | """ | ||
345 | |||
346 | # Call the base class constructor first | ||
347 | _FilemapBase.__init__(self, image, log) | ||
348 | self._log.debug("FilemapFiemap: initializing") | ||
349 | |||
350 | self._buf_size = _FIEMAP_BUFFER_SIZE | 138 | self._buf_size = _FIEMAP_BUFFER_SIZE |
351 | 139 | ||
352 | # Calculate how many 'struct fiemap_extent' elements fit the buffer | 140 | # Calculate how many 'struct fiemap_extent' elements fit the buffer |
@@ -362,6 +150,21 @@ class FilemapFiemap(_FilemapBase): | |||
362 | # Check if the FIEMAP ioctl is supported | 150 | # Check if the FIEMAP ioctl is supported |
363 | self.block_is_mapped(0) | 151 | self.block_is_mapped(0) |
364 | 152 | ||
153 | def __del__(self): | ||
154 | """The class destructor which just closes the image file.""" | ||
155 | if self._f_image_needs_close: | ||
156 | self._f_image.close() | ||
157 | |||
158 | def _open_image_file(self): | ||
159 | """Open the image file.""" | ||
160 | try: | ||
161 | self._f_image = open(self._image_path, 'rb') | ||
162 | except IOError as err: | ||
163 | raise Error("cannot open image file '%s': %s" | ||
164 | % (self._image_path, err)) | ||
165 | |||
166 | self._f_image_needs_close = True | ||
167 | |||
365 | def _invoke_fiemap(self, block, count): | 168 | def _invoke_fiemap(self, block, count): |
366 | """ | 169 | """ |
367 | Invoke the FIEMAP ioctl for 'count' blocks of the file starting from | 170 | Invoke the FIEMAP ioctl for 'count' blocks of the file starting from |
@@ -515,24 +318,10 @@ class FilemapFiemap(_FilemapBase): | |||
515 | % (hole_first, start + count - 1)) | 318 | % (hole_first, start + count - 1)) |
516 | yield (hole_first, start + count - 1) | 319 | yield (hole_first, start + count - 1) |
517 | 320 | ||
518 | def filemap(image, log=None): | ||
519 | """ | ||
520 | Create and return an instance of a Filemap class - 'FilemapFiemap' or | ||
521 | 'FilemapSeek', depending on what the system we run on supports. If the | ||
522 | FIEMAP ioctl is supported, an instance of the 'FilemapFiemap' class is | ||
523 | returned. Otherwise, if 'SEEK_HOLE' is supported an instance of the | ||
524 | 'FilemapSeek' class is returned. If none of these are supported, the | ||
525 | function generates an 'Error' type exception. | ||
526 | """ | ||
527 | |||
528 | try: | ||
529 | return FilemapFiemap(image, log) | ||
530 | except ErrorNotSupp: | ||
531 | return FilemapSeek(image, log) | ||
532 | 321 | ||
533 | def sparse_copy(src_fname, dst_fname, offset=0, skip=0): | 322 | def sparse_copy(src_fname, dst_fname, offset=0, skip=0): |
534 | """Efficiently copy sparse file to or into another file.""" | 323 | """Efficiently copy sparse file to or into another file.""" |
535 | fmap = filemap(src_fname) | 324 | fmap = FilemapFiemap(src_fname) |
536 | try: | 325 | try: |
537 | dst_file = open(dst_fname, 'r+b') | 326 | dst_file = open(dst_fname, 'r+b') |
538 | except IOError: | 327 | except IOError: |