diff options
| author | Ed Bartosh <ed.bartosh@linux.intel.com> | 2017-03-26 20:08:46 +0300 |
|---|---|---|
| committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2017-04-01 08:14:57 +0100 |
| commit | 6b80c13f7a82a312a3b981de5a56c66466ba1fac (patch) | |
| tree | eaa6074019745c74366fad7c6f6017666f4f7d83 /scripts/lib/wic/filemap.py | |
| parent | fae19c345de67663f577b2fc9e93070ea976f211 (diff) | |
| download | poky-6b80c13f7a82a312a3b981de5a56c66466ba1fac.tar.gz | |
filemap: remove FilemapSeek class
FIEMAP API was added to Linux kernel 2.6.28 back in 2008
SEEK_HOLE and SEEK_DATA API was added much letter.
As FIEMAP is used by filemap module as a default API it's
safe to remove FileMpSeek class as it's never used.
[YOCTO #10618]
(From OE-Core rev: 44e9406ea6e3263d2fb95e9d534a21f74f318480)
Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/wic/filemap.py')
| -rw-r--r-- | scripts/lib/wic/filemap.py | 309 |
1 files changed, 49 insertions, 260 deletions
diff --git a/scripts/lib/wic/filemap.py b/scripts/lib/wic/filemap.py index 080668e7c2..34523c02b6 100644 --- a/scripts/lib/wic/filemap.py +++ b/scripts/lib/wic/filemap.py | |||
| @@ -54,24 +54,46 @@ class Error(Exception): | |||
| 54 | """A class for all the other exceptions raised by this module.""" | 54 | """A class for all the other exceptions raised by this module.""" |
| 55 | pass | 55 | pass |
| 56 | 56 | ||
| 57 | # Below goes the FIEMAP ioctl implementation, which is not very readable | ||
| 58 | # because it deals with the rather complex FIEMAP ioctl. To understand the | ||
| 59 | # code, you need to know the FIEMAP interface, which is documented in the | ||
| 60 | # "Documentation/filesystems/fiemap.txt" file in the Linux kernel sources. | ||
| 61 | |||
| 62 | # Format string for 'struct fiemap' | ||
| 63 | _FIEMAP_FORMAT = "=QQLLLL" | ||
| 64 | # sizeof(struct fiemap) | ||
| 65 | _FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT) | ||
| 66 | # Format string for 'struct fiemap_extent' | ||
| 67 | _FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL" | ||
| 68 | # sizeof(struct fiemap_extent) | ||
| 69 | _FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT) | ||
| 70 | # The FIEMAP ioctl number | ||
| 71 | _FIEMAP_IOCTL = 0xC020660B | ||
| 72 | # This FIEMAP ioctl flag which instructs the kernel to sync the file before | ||
| 73 | # reading the block map | ||
| 74 | _FIEMAP_FLAG_SYNC = 0x00000001 | ||
| 75 | # Size of the buffer for 'struct fiemap_extent' elements which will be used | ||
| 76 | # when invoking the FIEMAP ioctl. The larger is the buffer, the less times the | ||
| 77 | # FIEMAP ioctl will be invoked. | ||
| 78 | _FIEMAP_BUFFER_SIZE = 256 * 1024 | ||
| 57 | 79 | ||
| 58 | class _FilemapBase(object): | 80 | class FilemapFiemap: |
| 59 | """ | 81 | """ |
| 60 | This is a base class for a couple of other classes in this module. This | 82 | This class provides API to the FIEMAP ioctl. Namely, it allows to iterate |
| 61 | class simply performs the common parts of the initialization process: opens | 83 | over all mapped blocks and over all holes. |
| 62 | the image file, gets its size, etc. The 'log' parameter is the logger object | 84 | |
| 63 | to use for printing messages. | 85 | This class synchronizes the image file every time it invokes the FIEMAP |
| 86 | ioctl in order to work-around early FIEMAP implementation kernel bugs. | ||
| 64 | """ | 87 | """ |
| 65 | 88 | ||
| 66 | def __init__(self, image, log=None): | 89 | def __init__(self, image): |
| 67 | """ | 90 | """ |
| 68 | Initialize a class instance. The 'image' argument is full path to the | 91 | Initialize a class instance. The 'image' argument is full the file |
| 69 | file or file object to operate on. | 92 | object to operate on. |
| 70 | """ | 93 | """ |
| 94 | self._log = logging.getLogger(__name__) | ||
| 71 | 95 | ||
| 72 | self._log = log | 96 | self._log.debug("FilemapFiemap: initializing") |
| 73 | if self._log is None: | ||
| 74 | self._log = logging.getLogger(__name__) | ||
| 75 | 97 | ||
| 76 | self._f_image_needs_close = False | 98 | self._f_image_needs_close = False |
| 77 | 99 | ||
| @@ -113,240 +135,6 @@ class _FilemapBase(object): | |||
| 113 | self._log.debug("block size %d, blocks count %d, image size %d" | 135 | self._log.debug("block size %d, blocks count %d, image size %d" |
| 114 | % (self.block_size, self.blocks_cnt, self.image_size)) | 136 | % (self.block_size, self.blocks_cnt, self.image_size)) |
| 115 | 137 | ||
| 116 | def __del__(self): | ||
| 117 | """The class destructor which just closes the image file.""" | ||
| 118 | if self._f_image_needs_close: | ||
| 119 | self._f_image.close() | ||
| 120 | |||
| 121 | def _open_image_file(self): | ||
| 122 | """Open the image file.""" | ||
| 123 | try: | ||
| 124 | self._f_image = open(self._image_path, 'rb') | ||
| 125 | except IOError as err: | ||
| 126 | raise Error("cannot open image file '%s': %s" | ||
| 127 | % (self._image_path, err)) | ||
| 128 | |||
| 129 | self._f_image_needs_close = True | ||
| 130 | |||
| 131 | def block_is_mapped(self, block): # pylint: disable=W0613,R0201 | ||
| 132 | """ | ||
| 133 | This method has has to be implemented by child classes. It returns | ||
| 134 | 'True' if block number 'block' of the image file is mapped and 'False' | ||
| 135 | otherwise. | ||
| 136 | """ | ||
| 137 | |||
| 138 | raise Error("the method is not implemented") | ||
| 139 | |||
| 140 | def block_is_unmapped(self, block): # pylint: disable=W0613,R0201 | ||
| 141 | """ | ||
| 142 | This method has has to be implemented by child classes. It returns | ||
| 143 | 'True' if block number 'block' of the image file is not mapped (hole) | ||
| 144 | and 'False' otherwise. | ||
| 145 | """ | ||
| 146 | |||
| 147 | raise Error("the method is not implemented") | ||
| 148 | |||
| 149 | def get_mapped_ranges(self, start, count): # pylint: disable=W0613,R0201 | ||
| 150 | """ | ||
| 151 | This method has has to be implemented by child classes. This is a | ||
| 152 | generator which yields ranges of mapped blocks in the file. The ranges | ||
| 153 | are tuples of 2 elements: [first, last], where 'first' is the first | ||
| 154 | mapped block and 'last' is the last mapped block. | ||
| 155 | |||
| 156 | The ranges are yielded for the area of the file of size 'count' blocks, | ||
| 157 | starting from block 'start'. | ||
| 158 | """ | ||
| 159 | |||
| 160 | raise Error("the method is not implemented") | ||
| 161 | |||
| 162 | def get_unmapped_ranges(self, start, count): # pylint: disable=W0613,R0201 | ||
| 163 | """ | ||
| 164 | This method has has to be implemented by child classes. Just like | ||
| 165 | 'get_mapped_ranges()', but yields unmapped block ranges instead | ||
| 166 | (holes). | ||
| 167 | """ | ||
| 168 | |||
| 169 | raise Error("the method is not implemented") | ||
| 170 | |||
| 171 | |||
| 172 | # The 'SEEK_HOLE' and 'SEEK_DATA' options of the file seek system call | ||
| 173 | _SEEK_DATA = 3 | ||
| 174 | _SEEK_HOLE = 4 | ||
| 175 | |||
| 176 | def _lseek(file_obj, offset, whence): | ||
| 177 | """This is a helper function which invokes 'os.lseek' for file object | ||
| 178 | 'file_obj' and with specified 'offset' and 'whence'. The 'whence' | ||
| 179 | argument is supposed to be either '_SEEK_DATA' or '_SEEK_HOLE'. When | ||
| 180 | there is no more data or hole starting from 'offset', this function | ||
| 181 | returns '-1'. Otherwise the data or hole position is returned.""" | ||
| 182 | |||
| 183 | try: | ||
| 184 | return os.lseek(file_obj.fileno(), offset, whence) | ||
| 185 | except OSError as err: | ||
| 186 | # The 'lseek' system call returns the ENXIO if there is no data or | ||
| 187 | # hole starting from the specified offset. | ||
| 188 | if err.errno == os.errno.ENXIO: | ||
| 189 | return -1 | ||
| 190 | elif err.errno == os.errno.EINVAL: | ||
| 191 | raise ErrorNotSupp("the kernel or file-system does not support " | ||
| 192 | "\"SEEK_HOLE\" and \"SEEK_DATA\"") | ||
| 193 | else: | ||
| 194 | raise | ||
| 195 | |||
| 196 | class FilemapSeek(_FilemapBase): | ||
| 197 | """ | ||
| 198 | This class uses the 'SEEK_HOLE' and 'SEEK_DATA' to find file block mapping. | ||
| 199 | Unfortunately, the current implementation requires the caller to have write | ||
| 200 | access to the image file. | ||
| 201 | """ | ||
| 202 | |||
| 203 | def __init__(self, image, log=None): | ||
| 204 | """Refer the '_FilemapBase' class for the documentation.""" | ||
| 205 | |||
| 206 | # Call the base class constructor first | ||
| 207 | _FilemapBase.__init__(self, image, log) | ||
| 208 | self._log.debug("FilemapSeek: initializing") | ||
| 209 | |||
| 210 | self._probe_seek_hole() | ||
| 211 | |||
| 212 | def _probe_seek_hole(self): | ||
| 213 | """ | ||
| 214 | Check whether the system implements 'SEEK_HOLE' and 'SEEK_DATA'. | ||
| 215 | Unfortunately, there seems to be no clean way for detecting this, | ||
| 216 | because often the system just fakes them by just assuming that all | ||
| 217 | files are fully mapped, so 'SEEK_HOLE' always returns EOF and | ||
| 218 | 'SEEK_DATA' always returns the requested offset. | ||
| 219 | |||
| 220 | I could not invent a better way of detecting the fake 'SEEK_HOLE' | ||
| 221 | implementation than just to create a temporary file in the same | ||
| 222 | directory where the image file resides. It would be nice to change this | ||
| 223 | to something better. | ||
| 224 | """ | ||
| 225 | |||
| 226 | directory = os.path.dirname(self._image_path) | ||
| 227 | |||
| 228 | try: | ||
| 229 | tmp_obj = tempfile.TemporaryFile("w+", dir=directory) | ||
| 230 | except IOError as err: | ||
| 231 | raise ErrorNotSupp("cannot create a temporary in \"%s\": %s" | ||
| 232 | % (directory, err)) | ||
| 233 | |||
| 234 | try: | ||
| 235 | os.ftruncate(tmp_obj.fileno(), self.block_size) | ||
| 236 | except OSError as err: | ||
| 237 | raise ErrorNotSupp("cannot truncate temporary file in \"%s\": %s" | ||
| 238 | % (directory, err)) | ||
| 239 | |||
| 240 | offs = _lseek(tmp_obj, 0, _SEEK_HOLE) | ||
| 241 | if offs != 0: | ||
| 242 | # We are dealing with the stub 'SEEK_HOLE' implementation which | ||
| 243 | # always returns EOF. | ||
| 244 | self._log.debug("lseek(0, SEEK_HOLE) returned %d" % offs) | ||
| 245 | raise ErrorNotSupp("the file-system does not support " | ||
| 246 | "\"SEEK_HOLE\" and \"SEEK_DATA\" but only " | ||
| 247 | "provides a stub implementation") | ||
| 248 | |||
| 249 | tmp_obj.close() | ||
| 250 | |||
| 251 | def block_is_mapped(self, block): | ||
| 252 | """Refer the '_FilemapBase' class for the documentation.""" | ||
| 253 | offs = _lseek(self._f_image, block * self.block_size, _SEEK_DATA) | ||
| 254 | if offs == -1: | ||
| 255 | result = False | ||
| 256 | else: | ||
| 257 | result = (offs // self.block_size == block) | ||
| 258 | |||
| 259 | self._log.debug("FilemapSeek: block_is_mapped(%d) returns %s" | ||
| 260 | % (block, result)) | ||
| 261 | return result | ||
| 262 | |||
| 263 | def block_is_unmapped(self, block): | ||
| 264 | """Refer the '_FilemapBase' class for the documentation.""" | ||
| 265 | return not self.block_is_mapped(block) | ||
| 266 | |||
| 267 | def _get_ranges(self, start, count, whence1, whence2): | ||
| 268 | """ | ||
| 269 | This function implements 'get_mapped_ranges()' and | ||
| 270 | 'get_unmapped_ranges()' depending on what is passed in the 'whence1' | ||
| 271 | and 'whence2' arguments. | ||
| 272 | """ | ||
| 273 | |||
| 274 | assert whence1 != whence2 | ||
| 275 | end = start * self.block_size | ||
| 276 | limit = end + count * self.block_size | ||
| 277 | |||
| 278 | while True: | ||
| 279 | start = _lseek(self._f_image, end, whence1) | ||
| 280 | if start == -1 or start >= limit or start == self.image_size: | ||
| 281 | break | ||
| 282 | |||
| 283 | end = _lseek(self._f_image, start, whence2) | ||
| 284 | if end == -1 or end == self.image_size: | ||
| 285 | end = self.blocks_cnt * self.block_size | ||
| 286 | if end > limit: | ||
| 287 | end = limit | ||
| 288 | |||
| 289 | start_blk = start // self.block_size | ||
| 290 | end_blk = end // self.block_size - 1 | ||
| 291 | self._log.debug("FilemapSeek: yielding range (%d, %d)" | ||
| 292 | % (start_blk, end_blk)) | ||
| 293 | yield (start_blk, end_blk) | ||
| 294 | |||
| 295 | def get_mapped_ranges(self, start, count): | ||
| 296 | """Refer the '_FilemapBase' class for the documentation.""" | ||
| 297 | self._log.debug("FilemapSeek: get_mapped_ranges(%d, %d(%d))" | ||
| 298 | % (start, count, start + count - 1)) | ||
| 299 | return self._get_ranges(start, count, _SEEK_DATA, _SEEK_HOLE) | ||
| 300 | |||
| 301 | def get_unmapped_ranges(self, start, count): | ||
| 302 | """Refer the '_FilemapBase' class for the documentation.""" | ||
| 303 | self._log.debug("FilemapSeek: get_unmapped_ranges(%d, %d(%d))" | ||
| 304 | % (start, count, start + count - 1)) | ||
| 305 | return self._get_ranges(start, count, _SEEK_HOLE, _SEEK_DATA) | ||
| 306 | |||
| 307 | |||
| 308 | # Below goes the FIEMAP ioctl implementation, which is not very readable | ||
| 309 | # because it deals with the rather complex FIEMAP ioctl. To understand the | ||
| 310 | # code, you need to know the FIEMAP interface, which is documented in the | ||
| 311 | # "Documentation/filesystems/fiemap.txt" file in the Linux kernel sources. | ||
| 312 | |||
| 313 | # Format string for 'struct fiemap' | ||
| 314 | _FIEMAP_FORMAT = "=QQLLLL" | ||
| 315 | # sizeof(struct fiemap) | ||
| 316 | _FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT) | ||
| 317 | # Format string for 'struct fiemap_extent' | ||
| 318 | _FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL" | ||
| 319 | # sizeof(struct fiemap_extent) | ||
| 320 | _FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT) | ||
| 321 | # The FIEMAP ioctl number | ||
| 322 | _FIEMAP_IOCTL = 0xC020660B | ||
| 323 | # This FIEMAP ioctl flag which instructs the kernel to sync the file before | ||
| 324 | # reading the block map | ||
| 325 | _FIEMAP_FLAG_SYNC = 0x00000001 | ||
| 326 | # Size of the buffer for 'struct fiemap_extent' elements which will be used | ||
| 327 | # when invoking the FIEMAP ioctl. The larger is the buffer, the less times the | ||
| 328 | # FIEMAP ioctl will be invoked. | ||
| 329 | _FIEMAP_BUFFER_SIZE = 256 * 1024 | ||
| 330 | |||
| 331 | class FilemapFiemap(_FilemapBase): | ||
| 332 | """ | ||
| 333 | This class provides API to the FIEMAP ioctl. Namely, it allows to iterate | ||
| 334 | over all mapped blocks and over all holes. | ||
| 335 | |||
| 336 | This class synchronizes the image file every time it invokes the FIEMAP | ||
| 337 | ioctl in order to work-around early FIEMAP implementation kernel bugs. | ||
| 338 | """ | ||
| 339 | |||
| 340 | def __init__(self, image, log=None): | ||
| 341 | """ | ||
| 342 | Initialize a class instance. The 'image' argument is full the file | ||
| 343 | object to operate on. | ||
| 344 | """ | ||
| 345 | |||
| 346 | # Call the base class constructor first | ||
| 347 | _FilemapBase.__init__(self, image, log) | ||
| 348 | self._log.debug("FilemapFiemap: initializing") | ||
| 349 | |||
| 350 | self._buf_size = _FIEMAP_BUFFER_SIZE | 138 | self._buf_size = _FIEMAP_BUFFER_SIZE |
| 351 | 139 | ||
| 352 | # Calculate how many 'struct fiemap_extent' elements fit the buffer | 140 | # Calculate how many 'struct fiemap_extent' elements fit the buffer |
| @@ -362,6 +150,21 @@ class FilemapFiemap(_FilemapBase): | |||
| 362 | # Check if the FIEMAP ioctl is supported | 150 | # Check if the FIEMAP ioctl is supported |
| 363 | self.block_is_mapped(0) | 151 | self.block_is_mapped(0) |
| 364 | 152 | ||
| 153 | def __del__(self): | ||
| 154 | """The class destructor which just closes the image file.""" | ||
| 155 | if self._f_image_needs_close: | ||
| 156 | self._f_image.close() | ||
| 157 | |||
| 158 | def _open_image_file(self): | ||
| 159 | """Open the image file.""" | ||
| 160 | try: | ||
| 161 | self._f_image = open(self._image_path, 'rb') | ||
| 162 | except IOError as err: | ||
| 163 | raise Error("cannot open image file '%s': %s" | ||
| 164 | % (self._image_path, err)) | ||
| 165 | |||
| 166 | self._f_image_needs_close = True | ||
| 167 | |||
| 365 | def _invoke_fiemap(self, block, count): | 168 | def _invoke_fiemap(self, block, count): |
| 366 | """ | 169 | """ |
| 367 | Invoke the FIEMAP ioctl for 'count' blocks of the file starting from | 170 | Invoke the FIEMAP ioctl for 'count' blocks of the file starting from |
| @@ -515,24 +318,10 @@ class FilemapFiemap(_FilemapBase): | |||
| 515 | % (hole_first, start + count - 1)) | 318 | % (hole_first, start + count - 1)) |
| 516 | yield (hole_first, start + count - 1) | 319 | yield (hole_first, start + count - 1) |
| 517 | 320 | ||
| 518 | def filemap(image, log=None): | ||
| 519 | """ | ||
| 520 | Create and return an instance of a Filemap class - 'FilemapFiemap' or | ||
| 521 | 'FilemapSeek', depending on what the system we run on supports. If the | ||
| 522 | FIEMAP ioctl is supported, an instance of the 'FilemapFiemap' class is | ||
| 523 | returned. Otherwise, if 'SEEK_HOLE' is supported an instance of the | ||
| 524 | 'FilemapSeek' class is returned. If none of these are supported, the | ||
| 525 | function generates an 'Error' type exception. | ||
| 526 | """ | ||
| 527 | |||
| 528 | try: | ||
| 529 | return FilemapFiemap(image, log) | ||
| 530 | except ErrorNotSupp: | ||
| 531 | return FilemapSeek(image, log) | ||
| 532 | 321 | ||
| 533 | def sparse_copy(src_fname, dst_fname, offset=0, skip=0): | 322 | def sparse_copy(src_fname, dst_fname, offset=0, skip=0): |
| 534 | """Efficiently copy sparse file to or into another file.""" | 323 | """Efficiently copy sparse file to or into another file.""" |
| 535 | fmap = filemap(src_fname) | 324 | fmap = FilemapFiemap(src_fname) |
| 536 | try: | 325 | try: |
| 537 | dst_file = open(dst_fname, 'r+b') | 326 | dst_file = open(dst_fname, 'r+b') |
| 538 | except IOError: | 327 | except IOError: |
