diff options
author | Adrian Dudau <adrian.dudau@enea.com> | 2013-12-12 13:38:32 +0100 |
---|---|---|
committer | Adrian Dudau <adrian.dudau@enea.com> | 2013-12-12 13:50:20 +0100 |
commit | e2e6f6fe07049f33cb6348780fa975162752e421 (patch) | |
tree | b1813295411235d1297a0ed642b1346b24fdfb12 /scripts/lib/mic/utils/fs_related.py | |
download | poky-e2e6f6fe07049f33cb6348780fa975162752e421.tar.gz |
initial commit of Enea Linux 3.1
Migrated from the internal git server on the dora-enea branch
Signed-off-by: Adrian Dudau <adrian.dudau@enea.com>
Diffstat (limited to 'scripts/lib/mic/utils/fs_related.py')
-rw-r--r-- | scripts/lib/mic/utils/fs_related.py | 1057 |
1 files changed, 1057 insertions, 0 deletions
diff --git a/scripts/lib/mic/utils/fs_related.py b/scripts/lib/mic/utils/fs_related.py new file mode 100644 index 0000000000..61617353eb --- /dev/null +++ b/scripts/lib/mic/utils/fs_related.py | |||
@@ -0,0 +1,1057 @@ | |||
1 | #!/usr/bin/python -tt | ||
2 | # | ||
3 | # Copyright (c) 2007, Red Hat, Inc. | ||
4 | # Copyright (c) 2009, 2010, 2011 Intel, Inc. | ||
5 | # | ||
6 | # This program is free software; you can redistribute it and/or modify it | ||
7 | # under the terms of the GNU General Public License as published by the Free | ||
8 | # Software Foundation; version 2 of the License | ||
9 | # | ||
10 | # This program is distributed in the hope that it will be useful, but | ||
11 | # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY | ||
12 | # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | ||
13 | # for more details. | ||
14 | # | ||
15 | # You should have received a copy of the GNU General Public License along | ||
16 | # with this program; if not, write to the Free Software Foundation, Inc., 59 | ||
17 | # Temple Place - Suite 330, Boston, MA 02111-1307, USA. | ||
18 | |||
19 | from __future__ import with_statement | ||
20 | import os | ||
21 | import sys | ||
22 | import errno | ||
23 | import stat | ||
24 | import random | ||
25 | import string | ||
26 | import time | ||
27 | import uuid | ||
28 | |||
29 | from mic import msger | ||
30 | from mic.utils import runner | ||
31 | from mic.utils.errors import * | ||
32 | from mic.utils.oe.misc import * | ||
33 | |||
34 | def find_binary_inchroot(binary, chroot): | ||
35 | paths = ["/usr/sbin", | ||
36 | "/usr/bin", | ||
37 | "/sbin", | ||
38 | "/bin" | ||
39 | ] | ||
40 | |||
41 | for path in paths: | ||
42 | bin_path = "%s/%s" % (path, binary) | ||
43 | if os.path.exists("%s/%s" % (chroot, bin_path)): | ||
44 | return bin_path | ||
45 | return None | ||
46 | |||
47 | def find_binary_path(binary): | ||
48 | if os.environ.has_key("PATH"): | ||
49 | paths = os.environ["PATH"].split(":") | ||
50 | else: | ||
51 | paths = [] | ||
52 | if os.environ.has_key("HOME"): | ||
53 | paths += [os.environ["HOME"] + "/bin"] | ||
54 | paths += ["/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"] | ||
55 | |||
56 | for path in paths: | ||
57 | bin_path = "%s/%s" % (path, binary) | ||
58 | if os.path.exists(bin_path): | ||
59 | return bin_path | ||
60 | raise CreatorError("Command '%s' is not available." % binary) | ||
61 | |||
62 | def makedirs(dirname): | ||
63 | """A version of os.makedirs() that doesn't throw an | ||
64 | exception if the leaf directory already exists. | ||
65 | """ | ||
66 | try: | ||
67 | os.makedirs(dirname) | ||
68 | except OSError, err: | ||
69 | if err.errno != errno.EEXIST: | ||
70 | raise | ||
71 | |||
72 | def mksquashfs(in_img, out_img): | ||
73 | fullpathmksquashfs = find_binary_path("mksquashfs") | ||
74 | args = [fullpathmksquashfs, in_img, out_img] | ||
75 | |||
76 | if not sys.stdout.isatty(): | ||
77 | args.append("-no-progress") | ||
78 | |||
79 | ret = runner.show(args) | ||
80 | if ret != 0: | ||
81 | raise SquashfsError("'%s' exited with error (%d)" % (' '.join(args), ret)) | ||
82 | |||
83 | def resize2fs(fs, size): | ||
84 | resize2fs = find_binary_path("resize2fs") | ||
85 | if size == 0: | ||
86 | # it means to minimalize it | ||
87 | return runner.show([resize2fs, '-M', fs]) | ||
88 | else: | ||
89 | return runner.show([resize2fs, fs, "%sK" % (size / 1024,)]) | ||
90 | |||
91 | def my_fuser(fp): | ||
92 | fuser = find_binary_path("fuser") | ||
93 | if not os.path.exists(fp): | ||
94 | return False | ||
95 | |||
96 | rc = runner.quiet([fuser, "-s", fp]) | ||
97 | if rc == 0: | ||
98 | for pid in runner.outs([fuser, fp]).split(): | ||
99 | fd = open("/proc/%s/cmdline" % pid, "r") | ||
100 | cmdline = fd.read() | ||
101 | fd.close() | ||
102 | if cmdline[:-1] == "/bin/bash": | ||
103 | return True | ||
104 | |||
105 | # not found | ||
106 | return False | ||
107 | |||
108 | class BindChrootMount: | ||
109 | """Represents a bind mount of a directory into a chroot.""" | ||
110 | def __init__(self, src, chroot, dest = None, option = None): | ||
111 | self.root = os.path.abspath(os.path.expanduser(chroot)) | ||
112 | self.option = option | ||
113 | |||
114 | self.orig_src = self.src = src | ||
115 | if os.path.islink(src): | ||
116 | self.src = os.readlink(src) | ||
117 | if not self.src.startswith('/'): | ||
118 | self.src = os.path.abspath(os.path.join(os.path.dirname(src), | ||
119 | self.src)) | ||
120 | |||
121 | if not dest: | ||
122 | dest = self.src | ||
123 | self.dest = os.path.join(self.root, dest.lstrip('/')) | ||
124 | |||
125 | self.mounted = False | ||
126 | self.mountcmd = find_binary_path("mount") | ||
127 | self.umountcmd = find_binary_path("umount") | ||
128 | |||
129 | def ismounted(self): | ||
130 | with open('/proc/mounts') as f: | ||
131 | for line in f: | ||
132 | if line.split()[1] == os.path.abspath(self.dest): | ||
133 | return True | ||
134 | |||
135 | return False | ||
136 | |||
137 | def has_chroot_instance(self): | ||
138 | lock = os.path.join(self.root, ".chroot.lock") | ||
139 | return my_fuser(lock) | ||
140 | |||
141 | def mount(self): | ||
142 | if self.mounted or self.ismounted(): | ||
143 | return | ||
144 | |||
145 | makedirs(self.dest) | ||
146 | rc = runner.show([self.mountcmd, "--bind", self.src, self.dest]) | ||
147 | if rc != 0: | ||
148 | raise MountError("Bind-mounting '%s' to '%s' failed" % | ||
149 | (self.src, self.dest)) | ||
150 | if self.option: | ||
151 | rc = runner.show([self.mountcmd, "--bind", "-o", "remount,%s" % self.option, self.dest]) | ||
152 | if rc != 0: | ||
153 | raise MountError("Bind-remounting '%s' failed" % self.dest) | ||
154 | |||
155 | self.mounted = True | ||
156 | if os.path.islink(self.orig_src): | ||
157 | dest = os.path.join(self.root, self.orig_src.lstrip('/')) | ||
158 | if not os.path.exists(dest): | ||
159 | os.symlink(self.src, dest) | ||
160 | |||
161 | def unmount(self): | ||
162 | if self.has_chroot_instance(): | ||
163 | return | ||
164 | |||
165 | if self.ismounted(): | ||
166 | runner.show([self.umountcmd, "-l", self.dest]) | ||
167 | self.mounted = False | ||
168 | |||
169 | class LoopbackMount: | ||
170 | """LoopbackMount compatibility layer for old API""" | ||
171 | def __init__(self, lofile, mountdir, fstype = None): | ||
172 | self.diskmount = DiskMount(LoopbackDisk(lofile,size = 0),mountdir,fstype,rmmountdir = True) | ||
173 | self.losetup = False | ||
174 | self.losetupcmd = find_binary_path("losetup") | ||
175 | |||
176 | def cleanup(self): | ||
177 | self.diskmount.cleanup() | ||
178 | |||
179 | def unmount(self): | ||
180 | self.diskmount.unmount() | ||
181 | |||
182 | def lounsetup(self): | ||
183 | if self.losetup: | ||
184 | runner.show([self.losetupcmd, "-d", self.loopdev]) | ||
185 | self.losetup = False | ||
186 | self.loopdev = None | ||
187 | |||
188 | def loopsetup(self): | ||
189 | if self.losetup: | ||
190 | return | ||
191 | |||
192 | self.loopdev = get_loop_device(self.losetupcmd, self.lofile) | ||
193 | self.losetup = True | ||
194 | |||
195 | def mount(self): | ||
196 | self.diskmount.mount() | ||
197 | |||
198 | class SparseLoopbackMount(LoopbackMount): | ||
199 | """SparseLoopbackMount compatibility layer for old API""" | ||
200 | def __init__(self, lofile, mountdir, size, fstype = None): | ||
201 | self.diskmount = DiskMount(SparseLoopbackDisk(lofile,size),mountdir,fstype,rmmountdir = True) | ||
202 | |||
203 | def expand(self, create = False, size = None): | ||
204 | self.diskmount.disk.expand(create, size) | ||
205 | |||
206 | def truncate(self, size = None): | ||
207 | self.diskmount.disk.truncate(size) | ||
208 | |||
209 | def create(self): | ||
210 | self.diskmount.disk.create() | ||
211 | |||
212 | class SparseExtLoopbackMount(SparseLoopbackMount): | ||
213 | """SparseExtLoopbackMount compatibility layer for old API""" | ||
214 | def __init__(self, lofile, mountdir, size, fstype, blocksize, fslabel): | ||
215 | self.diskmount = ExtDiskMount(SparseLoopbackDisk(lofile,size), mountdir, fstype, blocksize, fslabel, rmmountdir = True) | ||
216 | |||
217 | |||
218 | def __format_filesystem(self): | ||
219 | self.diskmount.__format_filesystem() | ||
220 | |||
221 | def create(self): | ||
222 | self.diskmount.disk.create() | ||
223 | |||
224 | def resize(self, size = None): | ||
225 | return self.diskmount.__resize_filesystem(size) | ||
226 | |||
227 | def mount(self): | ||
228 | self.diskmount.mount() | ||
229 | |||
230 | def __fsck(self): | ||
231 | self.extdiskmount.__fsck() | ||
232 | |||
233 | def __get_size_from_filesystem(self): | ||
234 | return self.diskmount.__get_size_from_filesystem() | ||
235 | |||
236 | def __resize_to_minimal(self): | ||
237 | return self.diskmount.__resize_to_minimal() | ||
238 | |||
239 | def resparse(self, size = None): | ||
240 | return self.diskmount.resparse(size) | ||
241 | |||
242 | class Disk: | ||
243 | """Generic base object for a disk | ||
244 | |||
245 | The 'create' method must make the disk visible as a block device - eg | ||
246 | by calling losetup. For RawDisk, this is obviously a no-op. The 'cleanup' | ||
247 | method must undo the 'create' operation. | ||
248 | """ | ||
249 | def __init__(self, size, device = None): | ||
250 | self._device = device | ||
251 | self._size = size | ||
252 | |||
253 | def create(self): | ||
254 | pass | ||
255 | |||
256 | def cleanup(self): | ||
257 | pass | ||
258 | |||
259 | def get_device(self): | ||
260 | return self._device | ||
261 | def set_device(self, path): | ||
262 | self._device = path | ||
263 | device = property(get_device, set_device) | ||
264 | |||
265 | def get_size(self): | ||
266 | return self._size | ||
267 | size = property(get_size) | ||
268 | |||
269 | |||
270 | class RawDisk(Disk): | ||
271 | """A Disk backed by a block device. | ||
272 | Note that create() is a no-op. | ||
273 | """ | ||
274 | def __init__(self, size, device): | ||
275 | Disk.__init__(self, size, device) | ||
276 | |||
277 | def fixed(self): | ||
278 | return True | ||
279 | |||
280 | def exists(self): | ||
281 | return True | ||
282 | |||
283 | |||
284 | class DiskImage(Disk): | ||
285 | """ | ||
286 | A Disk backed by a file. | ||
287 | """ | ||
288 | def __init__(self, image_file, size): | ||
289 | Disk.__init__(self, size) | ||
290 | self.image_file = image_file | ||
291 | |||
292 | def exists(self): | ||
293 | return os.path.exists(self.image_file) | ||
294 | |||
295 | def create(self): | ||
296 | if self.device is not None: | ||
297 | return | ||
298 | |||
299 | blocks = self.size / 1024 | ||
300 | if self.size - blocks * 1024: | ||
301 | blocks += 1 | ||
302 | |||
303 | # create disk image | ||
304 | dd_cmd = "dd if=/dev/zero of=%s bs=1024 seek=%d count=1" % \ | ||
305 | (self.image_file, blocks) | ||
306 | rc, out = exec_cmd(dd_cmd) | ||
307 | |||
308 | self.device = self.image_file | ||
309 | |||
310 | |||
311 | class LoopbackDisk(Disk): | ||
312 | """A Disk backed by a file via the loop module.""" | ||
313 | def __init__(self, lofile, size): | ||
314 | Disk.__init__(self, size) | ||
315 | self.lofile = lofile | ||
316 | self.losetupcmd = find_binary_path("losetup") | ||
317 | |||
318 | def fixed(self): | ||
319 | return False | ||
320 | |||
321 | def exists(self): | ||
322 | return os.path.exists(self.lofile) | ||
323 | |||
324 | def create(self): | ||
325 | if self.device is not None: | ||
326 | return | ||
327 | |||
328 | self.device = get_loop_device(self.losetupcmd, self.lofile) | ||
329 | |||
330 | def cleanup(self): | ||
331 | if self.device is None: | ||
332 | return | ||
333 | msger.debug("Losetup remove %s" % self.device) | ||
334 | rc = runner.show([self.losetupcmd, "-d", self.device]) | ||
335 | self.device = None | ||
336 | |||
337 | class SparseLoopbackDisk(LoopbackDisk): | ||
338 | """A Disk backed by a sparse file via the loop module.""" | ||
339 | def __init__(self, lofile, size): | ||
340 | LoopbackDisk.__init__(self, lofile, size) | ||
341 | |||
342 | def expand(self, create = False, size = None): | ||
343 | flags = os.O_WRONLY | ||
344 | if create: | ||
345 | flags |= os.O_CREAT | ||
346 | if not os.path.exists(self.lofile): | ||
347 | makedirs(os.path.dirname(self.lofile)) | ||
348 | |||
349 | if size is None: | ||
350 | size = self.size | ||
351 | |||
352 | msger.debug("Extending sparse file %s to %d" % (self.lofile, size)) | ||
353 | if create: | ||
354 | fd = os.open(self.lofile, flags, 0644) | ||
355 | else: | ||
356 | fd = os.open(self.lofile, flags) | ||
357 | |||
358 | if size <= 0: | ||
359 | size = 1 | ||
360 | try: | ||
361 | os.ftruncate(fd, size) | ||
362 | except: | ||
363 | # may be limited by 2G in 32bit env | ||
364 | os.ftruncate(fd, 2**31L) | ||
365 | |||
366 | os.close(fd) | ||
367 | |||
368 | def truncate(self, size = None): | ||
369 | if size is None: | ||
370 | size = self.size | ||
371 | |||
372 | msger.debug("Truncating sparse file %s to %d" % (self.lofile, size)) | ||
373 | fd = os.open(self.lofile, os.O_WRONLY) | ||
374 | os.ftruncate(fd, size) | ||
375 | os.close(fd) | ||
376 | |||
377 | def create(self): | ||
378 | self.expand(create = True) | ||
379 | LoopbackDisk.create(self) | ||
380 | |||
381 | class Mount: | ||
382 | """A generic base class to deal with mounting things.""" | ||
383 | def __init__(self, mountdir): | ||
384 | self.mountdir = mountdir | ||
385 | |||
386 | def cleanup(self): | ||
387 | self.unmount() | ||
388 | |||
389 | def mount(self, options = None): | ||
390 | pass | ||
391 | |||
392 | def unmount(self): | ||
393 | pass | ||
394 | |||
395 | class DiskMount(Mount): | ||
396 | """A Mount object that handles mounting of a Disk.""" | ||
397 | def __init__(self, disk, mountdir, fstype = None, rmmountdir = True): | ||
398 | Mount.__init__(self, mountdir) | ||
399 | |||
400 | self.disk = disk | ||
401 | self.fstype = fstype | ||
402 | self.rmmountdir = rmmountdir | ||
403 | |||
404 | self.mounted = False | ||
405 | self.rmdir = False | ||
406 | if fstype: | ||
407 | self.mkfscmd = find_binary_path("mkfs." + self.fstype) | ||
408 | else: | ||
409 | self.mkfscmd = None | ||
410 | self.mountcmd = find_binary_path("mount") | ||
411 | self.umountcmd = find_binary_path("umount") | ||
412 | |||
413 | def cleanup(self): | ||
414 | Mount.cleanup(self) | ||
415 | self.disk.cleanup() | ||
416 | |||
417 | def unmount(self): | ||
418 | if self.mounted: | ||
419 | msger.debug("Unmounting directory %s" % self.mountdir) | ||
420 | runner.quiet('sync') # sync the data on this mount point | ||
421 | rc = runner.show([self.umountcmd, "-l", self.mountdir]) | ||
422 | if rc == 0: | ||
423 | self.mounted = False | ||
424 | else: | ||
425 | raise MountError("Failed to umount %s" % self.mountdir) | ||
426 | if self.rmdir and not self.mounted: | ||
427 | try: | ||
428 | os.rmdir(self.mountdir) | ||
429 | except OSError, e: | ||
430 | pass | ||
431 | self.rmdir = False | ||
432 | |||
433 | |||
434 | def __create(self): | ||
435 | self.disk.create() | ||
436 | |||
437 | |||
438 | def mount(self, options = None): | ||
439 | if self.mounted: | ||
440 | return | ||
441 | |||
442 | if not os.path.isdir(self.mountdir): | ||
443 | msger.debug("Creating mount point %s" % self.mountdir) | ||
444 | os.makedirs(self.mountdir) | ||
445 | self.rmdir = self.rmmountdir | ||
446 | |||
447 | self.__create() | ||
448 | |||
449 | msger.debug("Mounting %s at %s" % (self.disk.device, self.mountdir)) | ||
450 | if options: | ||
451 | args = [ self.mountcmd, "-o", options, self.disk.device, self.mountdir ] | ||
452 | else: | ||
453 | args = [ self.mountcmd, self.disk.device, self.mountdir ] | ||
454 | if self.fstype: | ||
455 | args.extend(["-t", self.fstype]) | ||
456 | |||
457 | rc = runner.show(args) | ||
458 | if rc != 0: | ||
459 | raise MountError("Failed to mount '%s' to '%s' with command '%s'. Retval: %s" % | ||
460 | (self.disk.device, self.mountdir, " ".join(args), rc)) | ||
461 | |||
462 | self.mounted = True | ||
463 | |||
464 | class ExtDiskMount(DiskMount): | ||
465 | """A DiskMount object that is able to format/resize ext[23] filesystems.""" | ||
466 | def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None): | ||
467 | DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir) | ||
468 | self.blocksize = blocksize | ||
469 | self.fslabel = fslabel.replace("/", "") | ||
470 | self.uuid = str(uuid.uuid4()) | ||
471 | self.skipformat = skipformat | ||
472 | self.fsopts = fsopts | ||
473 | self.extopts = None | ||
474 | self.dumpe2fs = find_binary_path("dumpe2fs") | ||
475 | self.tune2fs = find_binary_path("tune2fs") | ||
476 | |||
477 | def __parse_field(self, output, field): | ||
478 | for line in output.split("\n"): | ||
479 | if line.startswith(field + ":"): | ||
480 | return line[len(field) + 1:].strip() | ||
481 | |||
482 | raise KeyError("Failed to find field '%s' in output" % field) | ||
483 | |||
484 | def __format_filesystem(self): | ||
485 | if self.skipformat: | ||
486 | msger.debug("Skip filesystem format.") | ||
487 | return | ||
488 | |||
489 | msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device)) | ||
490 | cmdlist = [self.mkfscmd, "-F", "-L", self.fslabel, "-m", "1", "-b", | ||
491 | str(self.blocksize), "-U", self.uuid] | ||
492 | if self.extopts: | ||
493 | cmdlist.extend(self.extopts.split()) | ||
494 | cmdlist.extend([self.disk.device]) | ||
495 | |||
496 | rc, errout = runner.runtool(cmdlist, catch=2) | ||
497 | if rc != 0: | ||
498 | raise MountError("Error creating %s filesystem on disk %s:\n%s" % | ||
499 | (self.fstype, self.disk.device, errout)) | ||
500 | |||
501 | if not self.extopts: | ||
502 | msger.debug("Tuning filesystem on %s" % self.disk.device) | ||
503 | runner.show([self.tune2fs, "-c0", "-i0", "-Odir_index", "-ouser_xattr,acl", self.disk.device]) | ||
504 | |||
505 | def __resize_filesystem(self, size = None): | ||
506 | current_size = os.stat(self.disk.lofile)[stat.ST_SIZE] | ||
507 | |||
508 | if size is None: | ||
509 | size = self.disk.size | ||
510 | |||
511 | if size == current_size: | ||
512 | return | ||
513 | |||
514 | if size > current_size: | ||
515 | self.disk.expand(size) | ||
516 | |||
517 | self.__fsck() | ||
518 | |||
519 | resize2fs(self.disk.lofile, size) | ||
520 | return size | ||
521 | |||
522 | def __create(self): | ||
523 | resize = False | ||
524 | if not self.disk.fixed() and self.disk.exists(): | ||
525 | resize = True | ||
526 | |||
527 | self.disk.create() | ||
528 | |||
529 | if resize: | ||
530 | self.__resize_filesystem() | ||
531 | else: | ||
532 | self.__format_filesystem() | ||
533 | |||
534 | def mount(self, options = None): | ||
535 | self.__create() | ||
536 | DiskMount.mount(self, options) | ||
537 | |||
538 | def __fsck(self): | ||
539 | msger.info("Checking filesystem %s" % self.disk.lofile) | ||
540 | runner.quiet(["/sbin/e2fsck", "-f", "-y", self.disk.lofile]) | ||
541 | |||
542 | def __get_size_from_filesystem(self): | ||
543 | return int(self.__parse_field(runner.outs([self.dumpe2fs, '-h', self.disk.lofile]), | ||
544 | "Block count")) * self.blocksize | ||
545 | |||
546 | def __resize_to_minimal(self): | ||
547 | self.__fsck() | ||
548 | |||
549 | # | ||
550 | # Use a binary search to find the minimal size | ||
551 | # we can resize the image to | ||
552 | # | ||
553 | bot = 0 | ||
554 | top = self.__get_size_from_filesystem() | ||
555 | while top != (bot + 1): | ||
556 | t = bot + ((top - bot) / 2) | ||
557 | |||
558 | if not resize2fs(self.disk.lofile, t): | ||
559 | top = t | ||
560 | else: | ||
561 | bot = t | ||
562 | return top | ||
563 | |||
564 | def resparse(self, size = None): | ||
565 | self.cleanup() | ||
566 | if size == 0: | ||
567 | minsize = 0 | ||
568 | else: | ||
569 | minsize = self.__resize_to_minimal() | ||
570 | self.disk.truncate(minsize) | ||
571 | |||
572 | self.__resize_filesystem(size) | ||
573 | return minsize | ||
574 | |||
575 | class VfatDiskMount(DiskMount): | ||
576 | """A DiskMount object that is able to format vfat/msdos filesystems.""" | ||
577 | def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None): | ||
578 | DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir) | ||
579 | self.blocksize = blocksize | ||
580 | self.fslabel = fslabel.replace("/", "") | ||
581 | rand1 = random.randint(0, 2**16 - 1) | ||
582 | rand2 = random.randint(0, 2**16 - 1) | ||
583 | self.uuid = "%04X-%04X" % (rand1, rand2) | ||
584 | self.skipformat = skipformat | ||
585 | self.fsopts = fsopts | ||
586 | self.fsckcmd = find_binary_path("fsck." + self.fstype) | ||
587 | |||
588 | def __format_filesystem(self): | ||
589 | if self.skipformat: | ||
590 | msger.debug("Skip filesystem format.") | ||
591 | return | ||
592 | |||
593 | msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device)) | ||
594 | rc = runner.show([self.mkfscmd, "-n", self.fslabel, | ||
595 | "-i", self.uuid.replace("-", ""), self.disk.device]) | ||
596 | if rc != 0: | ||
597 | raise MountError("Error creating %s filesystem on disk %s" % (self.fstype,self.disk.device)) | ||
598 | |||
599 | msger.verbose("Tuning filesystem on %s" % self.disk.device) | ||
600 | |||
601 | def __resize_filesystem(self, size = None): | ||
602 | current_size = os.stat(self.disk.lofile)[stat.ST_SIZE] | ||
603 | |||
604 | if size is None: | ||
605 | size = self.disk.size | ||
606 | |||
607 | if size == current_size: | ||
608 | return | ||
609 | |||
610 | if size > current_size: | ||
611 | self.disk.expand(size) | ||
612 | |||
613 | self.__fsck() | ||
614 | |||
615 | #resize2fs(self.disk.lofile, size) | ||
616 | return size | ||
617 | |||
618 | def __create(self): | ||
619 | resize = False | ||
620 | if not self.disk.fixed() and self.disk.exists(): | ||
621 | resize = True | ||
622 | |||
623 | self.disk.create() | ||
624 | |||
625 | if resize: | ||
626 | self.__resize_filesystem() | ||
627 | else: | ||
628 | self.__format_filesystem() | ||
629 | |||
630 | def mount(self, options = None): | ||
631 | self.__create() | ||
632 | DiskMount.mount(self, options) | ||
633 | |||
634 | def __fsck(self): | ||
635 | msger.debug("Checking filesystem %s" % self.disk.lofile) | ||
636 | runner.show([self.fsckcmd, "-y", self.disk.lofile]) | ||
637 | |||
638 | def __get_size_from_filesystem(self): | ||
639 | return self.disk.size | ||
640 | |||
641 | def __resize_to_minimal(self): | ||
642 | self.__fsck() | ||
643 | |||
644 | # | ||
645 | # Use a binary search to find the minimal size | ||
646 | # we can resize the image to | ||
647 | # | ||
648 | bot = 0 | ||
649 | top = self.__get_size_from_filesystem() | ||
650 | return top | ||
651 | |||
652 | def resparse(self, size = None): | ||
653 | self.cleanup() | ||
654 | minsize = self.__resize_to_minimal() | ||
655 | self.disk.truncate(minsize) | ||
656 | self.__resize_filesystem(size) | ||
657 | return minsize | ||
658 | |||
659 | class BtrfsDiskMount(DiskMount): | ||
660 | """A DiskMount object that is able to format/resize btrfs filesystems.""" | ||
661 | def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None): | ||
662 | self.__check_btrfs() | ||
663 | DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir) | ||
664 | self.blocksize = blocksize | ||
665 | self.fslabel = fslabel.replace("/", "") | ||
666 | self.uuid = None | ||
667 | self.skipformat = skipformat | ||
668 | self.fsopts = fsopts | ||
669 | self.blkidcmd = find_binary_path("blkid") | ||
670 | self.btrfsckcmd = find_binary_path("btrfsck") | ||
671 | |||
672 | def __check_btrfs(self): | ||
673 | found = False | ||
674 | """ Need to load btrfs module to mount it """ | ||
675 | load_module("btrfs") | ||
676 | for line in open("/proc/filesystems").xreadlines(): | ||
677 | if line.find("btrfs") > -1: | ||
678 | found = True | ||
679 | break | ||
680 | if not found: | ||
681 | raise MountError("Your system can't mount btrfs filesystem, please make sure your kernel has btrfs support and the module btrfs.ko has been loaded.") | ||
682 | |||
683 | # disable selinux, selinux will block write | ||
684 | if os.path.exists("/usr/sbin/setenforce"): | ||
685 | runner.show(["/usr/sbin/setenforce", "0"]) | ||
686 | |||
687 | def __parse_field(self, output, field): | ||
688 | for line in output.split(" "): | ||
689 | if line.startswith(field + "="): | ||
690 | return line[len(field) + 1:].strip().replace("\"", "") | ||
691 | |||
692 | raise KeyError("Failed to find field '%s' in output" % field) | ||
693 | |||
694 | def __format_filesystem(self): | ||
695 | if self.skipformat: | ||
696 | msger.debug("Skip filesystem format.") | ||
697 | return | ||
698 | |||
699 | msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device)) | ||
700 | rc = runner.show([self.mkfscmd, "-L", self.fslabel, self.disk.device]) | ||
701 | if rc != 0: | ||
702 | raise MountError("Error creating %s filesystem on disk %s" % (self.fstype,self.disk.device)) | ||
703 | |||
704 | self.uuid = self.__parse_field(runner.outs([self.blkidcmd, self.disk.device]), "UUID") | ||
705 | |||
706 | def __resize_filesystem(self, size = None): | ||
707 | current_size = os.stat(self.disk.lofile)[stat.ST_SIZE] | ||
708 | |||
709 | if size is None: | ||
710 | size = self.disk.size | ||
711 | |||
712 | if size == current_size: | ||
713 | return | ||
714 | |||
715 | if size > current_size: | ||
716 | self.disk.expand(size) | ||
717 | |||
718 | self.__fsck() | ||
719 | return size | ||
720 | |||
721 | def __create(self): | ||
722 | resize = False | ||
723 | if not self.disk.fixed() and self.disk.exists(): | ||
724 | resize = True | ||
725 | |||
726 | self.disk.create() | ||
727 | |||
728 | if resize: | ||
729 | self.__resize_filesystem() | ||
730 | else: | ||
731 | self.__format_filesystem() | ||
732 | |||
733 | def mount(self, options = None): | ||
734 | self.__create() | ||
735 | DiskMount.mount(self, options) | ||
736 | |||
737 | def __fsck(self): | ||
738 | msger.debug("Checking filesystem %s" % self.disk.lofile) | ||
739 | runner.quiet([self.btrfsckcmd, self.disk.lofile]) | ||
740 | |||
741 | def __get_size_from_filesystem(self): | ||
742 | return self.disk.size | ||
743 | |||
744 | def __resize_to_minimal(self): | ||
745 | self.__fsck() | ||
746 | |||
747 | return self.__get_size_from_filesystem() | ||
748 | |||
749 | def resparse(self, size = None): | ||
750 | self.cleanup() | ||
751 | minsize = self.__resize_to_minimal() | ||
752 | self.disk.truncate(minsize) | ||
753 | self.__resize_filesystem(size) | ||
754 | return minsize | ||
755 | |||
756 | class DeviceMapperSnapshot(object): | ||
757 | def __init__(self, imgloop, cowloop): | ||
758 | self.imgloop = imgloop | ||
759 | self.cowloop = cowloop | ||
760 | |||
761 | self.__created = False | ||
762 | self.__name = None | ||
763 | self.dmsetupcmd = find_binary_path("dmsetup") | ||
764 | |||
765 | """Load dm_snapshot if it isn't loaded""" | ||
766 | load_module("dm_snapshot") | ||
767 | |||
768 | def get_path(self): | ||
769 | if self.__name is None: | ||
770 | return None | ||
771 | return os.path.join("/dev/mapper", self.__name) | ||
772 | path = property(get_path) | ||
773 | |||
774 | def create(self): | ||
775 | if self.__created: | ||
776 | return | ||
777 | |||
778 | self.imgloop.create() | ||
779 | self.cowloop.create() | ||
780 | |||
781 | self.__name = "imgcreate-%d-%d" % (os.getpid(), | ||
782 | random.randint(0, 2**16)) | ||
783 | |||
784 | size = os.stat(self.imgloop.lofile)[stat.ST_SIZE] | ||
785 | |||
786 | table = "0 %d snapshot %s %s p 8" % (size / 512, | ||
787 | self.imgloop.device, | ||
788 | self.cowloop.device) | ||
789 | |||
790 | args = [self.dmsetupcmd, "create", self.__name, "--table", table] | ||
791 | if runner.show(args) != 0: | ||
792 | self.cowloop.cleanup() | ||
793 | self.imgloop.cleanup() | ||
794 | raise SnapshotError("Could not create snapshot device using: " + ' '.join(args)) | ||
795 | |||
796 | self.__created = True | ||
797 | |||
798 | def remove(self, ignore_errors = False): | ||
799 | if not self.__created: | ||
800 | return | ||
801 | |||
802 | time.sleep(2) | ||
803 | rc = runner.show([self.dmsetupcmd, "remove", self.__name]) | ||
804 | if not ignore_errors and rc != 0: | ||
805 | raise SnapshotError("Could not remove snapshot device") | ||
806 | |||
807 | self.__name = None | ||
808 | self.__created = False | ||
809 | |||
810 | self.cowloop.cleanup() | ||
811 | self.imgloop.cleanup() | ||
812 | |||
813 | def get_cow_used(self): | ||
814 | if not self.__created: | ||
815 | return 0 | ||
816 | |||
817 | # | ||
818 | # dmsetup status on a snapshot returns e.g. | ||
819 | # "0 8388608 snapshot 416/1048576" | ||
820 | # or, more generally: | ||
821 | # "A B snapshot C/D" | ||
822 | # where C is the number of 512 byte sectors in use | ||
823 | # | ||
824 | out = runner.outs([self.dmsetupcmd, "status", self.__name]) | ||
825 | try: | ||
826 | return int((out.split()[3]).split('/')[0]) * 512 | ||
827 | except ValueError: | ||
828 | raise SnapshotError("Failed to parse dmsetup status: " + out) | ||
829 | |||
830 | def create_image_minimizer(path, image, minimal_size): | ||
831 | """ | ||
832 | Builds a copy-on-write image which can be used to | ||
833 | create a device-mapper snapshot of an image where | ||
834 | the image's filesystem is as small as possible | ||
835 | |||
836 | The steps taken are: | ||
837 | 1) Create a sparse COW | ||
838 | 2) Loopback mount the image and the COW | ||
839 | 3) Create a device-mapper snapshot of the image | ||
840 | using the COW | ||
841 | 4) Resize the filesystem to the minimal size | ||
842 | 5) Determine the amount of space used in the COW | ||
843 | 6) Restroy the device-mapper snapshot | ||
844 | 7) Truncate the COW, removing unused space | ||
845 | 8) Create a squashfs of the COW | ||
846 | """ | ||
847 | imgloop = LoopbackDisk(image, None) # Passing bogus size - doesn't matter | ||
848 | |||
849 | cowloop = SparseLoopbackDisk(os.path.join(os.path.dirname(path), "osmin"), | ||
850 | 64L * 1024L * 1024L) | ||
851 | |||
852 | snapshot = DeviceMapperSnapshot(imgloop, cowloop) | ||
853 | |||
854 | try: | ||
855 | snapshot.create() | ||
856 | |||
857 | resize2fs(snapshot.path, minimal_size) | ||
858 | |||
859 | cow_used = snapshot.get_cow_used() | ||
860 | finally: | ||
861 | snapshot.remove(ignore_errors = (not sys.exc_info()[0] is None)) | ||
862 | |||
863 | cowloop.truncate(cow_used) | ||
864 | |||
865 | mksquashfs(cowloop.lofile, path) | ||
866 | |||
867 | os.unlink(cowloop.lofile) | ||
868 | |||
869 | def load_module(module): | ||
870 | found = False | ||
871 | for line in open('/proc/modules').xreadlines(): | ||
872 | if line.startswith("%s " % module): | ||
873 | found = True | ||
874 | break | ||
875 | if not found: | ||
876 | msger.info("Loading %s..." % module) | ||
877 | runner.quiet(['modprobe', module]) | ||
878 | |||
879 | class LoopDevice(object): | ||
880 | def __init__(self, loopid=None): | ||
881 | self.device = None | ||
882 | self.loopid = loopid | ||
883 | self.created = False | ||
884 | self.kpartxcmd = find_binary_path("kpartx") | ||
885 | self.losetupcmd = find_binary_path("losetup") | ||
886 | |||
887 | def register(self, device): | ||
888 | self.device = device | ||
889 | self.loopid = None | ||
890 | self.created = True | ||
891 | |||
892 | def reg_atexit(self): | ||
893 | import atexit | ||
894 | atexit.register(self.close) | ||
895 | |||
896 | def _genloopid(self): | ||
897 | import glob | ||
898 | if not glob.glob("/dev/loop[0-9]*"): | ||
899 | return 10 | ||
900 | |||
901 | fint = lambda x: x[9:].isdigit() and int(x[9:]) or 0 | ||
902 | maxid = 1 + max(filter(lambda x: x<100, | ||
903 | map(fint, glob.glob("/dev/loop[0-9]*")))) | ||
904 | if maxid < 10: maxid = 10 | ||
905 | if maxid >= 100: raise | ||
906 | return maxid | ||
907 | |||
908 | def _kpseek(self, device): | ||
909 | rc, out = runner.runtool([self.kpartxcmd, '-l', '-v', device]) | ||
910 | if rc != 0: | ||
911 | raise MountError("Can't query dm snapshot on %s" % device) | ||
912 | for line in out.splitlines(): | ||
913 | if line and line.startswith("loop"): | ||
914 | return True | ||
915 | return False | ||
916 | |||
917 | def _loseek(self, device): | ||
918 | import re | ||
919 | rc, out = runner.runtool([self.losetupcmd, '-a']) | ||
920 | if rc != 0: | ||
921 | raise MountError("Failed to run 'losetup -a'") | ||
922 | for line in out.splitlines(): | ||
923 | m = re.match("([^:]+): .*", line) | ||
924 | if m and m.group(1) == device: | ||
925 | return True | ||
926 | return False | ||
927 | |||
928 | def create(self): | ||
929 | if not self.created: | ||
930 | if not self.loopid: | ||
931 | self.loopid = self._genloopid() | ||
932 | self.device = "/dev/loop%d" % self.loopid | ||
933 | if os.path.exists(self.device): | ||
934 | if self._loseek(self.device): | ||
935 | raise MountError("Device busy: %s" % self.device) | ||
936 | else: | ||
937 | self.created = True | ||
938 | return | ||
939 | |||
940 | mknod = find_binary_path('mknod') | ||
941 | rc = runner.show([mknod, '-m664', self.device, 'b', '7', str(self.loopid)]) | ||
942 | if rc != 0: | ||
943 | raise MountError("Failed to create device %s" % self.device) | ||
944 | else: | ||
945 | self.created = True | ||
946 | |||
947 | def close(self): | ||
948 | if self.created: | ||
949 | try: | ||
950 | self.cleanup() | ||
951 | self.device = None | ||
952 | except MountError, e: | ||
953 | msger.error("%s" % e) | ||
954 | |||
955 | def cleanup(self): | ||
956 | |||
957 | if self.device is None: | ||
958 | return | ||
959 | |||
960 | |||
961 | if self._kpseek(self.device): | ||
962 | if self.created: | ||
963 | for i in range(3, os.sysconf("SC_OPEN_MAX")): | ||
964 | try: | ||
965 | os.close(i) | ||
966 | except: | ||
967 | pass | ||
968 | runner.quiet([self.kpartxcmd, "-d", self.device]) | ||
969 | if self._loseek(self.device): | ||
970 | runner.quiet([self.losetupcmd, "-d", self.device]) | ||
971 | # FIXME: should sleep a while between two loseek | ||
972 | if self._loseek(self.device): | ||
973 | msger.warning("Can't cleanup loop device %s" % self.device) | ||
974 | elif self.loopid: | ||
975 | os.unlink(self.device) | ||
976 | |||
977 | DEVICE_PIDFILE_DIR = "/var/tmp/mic/device" | ||
978 | DEVICE_LOCKFILE = "/var/lock/__mic_loopdev.lock" | ||
979 | |||
980 | def get_loop_device(losetupcmd, lofile): | ||
981 | global DEVICE_PIDFILE_DIR | ||
982 | global DEVICE_LOCKFILE | ||
983 | |||
984 | import fcntl | ||
985 | makedirs(os.path.dirname(DEVICE_LOCKFILE)) | ||
986 | fp = open(DEVICE_LOCKFILE, 'w') | ||
987 | fcntl.flock(fp, fcntl.LOCK_EX) | ||
988 | try: | ||
989 | loopdev = None | ||
990 | devinst = LoopDevice() | ||
991 | |||
992 | # clean up left loop device first | ||
993 | clean_loop_devices() | ||
994 | |||
995 | # provide an avaible loop device | ||
996 | rc, out = runner.runtool([losetupcmd, "--find"]) | ||
997 | if rc == 0: | ||
998 | loopdev = out.split()[0] | ||
999 | devinst.register(loopdev) | ||
1000 | if not loopdev or not os.path.exists(loopdev): | ||
1001 | devinst.create() | ||
1002 | loopdev = devinst.device | ||
1003 | |||
1004 | # setup a loop device for image file | ||
1005 | rc = runner.show([losetupcmd, loopdev, lofile]) | ||
1006 | if rc != 0: | ||
1007 | raise MountError("Failed to setup loop device for '%s'" % lofile) | ||
1008 | |||
1009 | devinst.reg_atexit() | ||
1010 | |||
1011 | # try to save device and pid | ||
1012 | makedirs(DEVICE_PIDFILE_DIR) | ||
1013 | pidfile = os.path.join(DEVICE_PIDFILE_DIR, os.path.basename(loopdev)) | ||
1014 | if os.path.exists(pidfile): | ||
1015 | os.unlink(pidfile) | ||
1016 | with open(pidfile, 'w') as wf: | ||
1017 | wf.write(str(os.getpid())) | ||
1018 | |||
1019 | except MountError, err: | ||
1020 | raise CreatorError("%s" % str(err)) | ||
1021 | except: | ||
1022 | raise | ||
1023 | finally: | ||
1024 | try: | ||
1025 | fcntl.flock(fp, fcntl.LOCK_UN) | ||
1026 | fp.close() | ||
1027 | os.unlink(DEVICE_LOCKFILE) | ||
1028 | except: | ||
1029 | pass | ||
1030 | |||
1031 | return loopdev | ||
1032 | |||
1033 | def clean_loop_devices(piddir=DEVICE_PIDFILE_DIR): | ||
1034 | if not os.path.exists(piddir) or not os.path.isdir(piddir): | ||
1035 | return | ||
1036 | |||
1037 | for loopdev in os.listdir(piddir): | ||
1038 | pidfile = os.path.join(piddir, loopdev) | ||
1039 | try: | ||
1040 | with open(pidfile, 'r') as rf: | ||
1041 | devpid = int(rf.read()) | ||
1042 | except: | ||
1043 | devpid = None | ||
1044 | |||
1045 | # if the process using this device is alive, skip it | ||
1046 | if not devpid or os.path.exists(os.path.join('/proc', str(devpid))): | ||
1047 | continue | ||
1048 | |||
1049 | # try to clean it up | ||
1050 | try: | ||
1051 | devinst = LoopDevice() | ||
1052 | devinst.register(os.path.join('/dev', loopdev)) | ||
1053 | devinst.cleanup() | ||
1054 | os.unlink(pidfile) | ||
1055 | except: | ||
1056 | pass | ||
1057 | |||