diff options
-rw-r--r-- | scripts/lib/mic/utils/partitionedfs.py | 355 |
1 files changed, 2 insertions, 353 deletions
diff --git a/scripts/lib/mic/utils/partitionedfs.py b/scripts/lib/mic/utils/partitionedfs.py index 0c4c9ecfa0..ef92125c98 100644 --- a/scripts/lib/mic/utils/partitionedfs.py +++ b/scripts/lib/mic/utils/partitionedfs.py | |||
@@ -33,18 +33,13 @@ MBR_OVERHEAD = 1 | |||
33 | SECTOR_SIZE = 512 | 33 | SECTOR_SIZE = 512 |
34 | 34 | ||
35 | class PartitionedMount(Mount): | 35 | class PartitionedMount(Mount): |
36 | def __init__(self, mountdir, skipformat = False): | 36 | def __init__(self, mountdir): |
37 | Mount.__init__(self, mountdir) | 37 | Mount.__init__(self, mountdir) |
38 | self.disks = {} | 38 | self.disks = {} |
39 | self.partitions = [] | 39 | self.partitions = [] |
40 | self.subvolumes = [] | ||
41 | self.mapped = False | ||
42 | self.mountOrder = [] | 40 | self.mountOrder = [] |
43 | self.unmountOrder = [] | 41 | self.unmountOrder = [] |
44 | self.parted = find_binary_path("parted") | 42 | self.parted = find_binary_path("parted") |
45 | self.btrfscmd=None | ||
46 | self.skipformat = skipformat | ||
47 | self.snapshot_created = self.skipformat | ||
48 | # Size of a sector used in calculations | 43 | # Size of a sector used in calculations |
49 | self.sector_size = SECTOR_SIZE | 44 | self.sector_size = SECTOR_SIZE |
50 | self._partitions_layed_out = False | 45 | self._partitions_layed_out = False |
@@ -62,7 +57,6 @@ class PartitionedMount(Mount): | |||
62 | 57 | ||
63 | self.disks[disk_name] = \ | 58 | self.disks[disk_name] = \ |
64 | { 'disk': None, # Disk object | 59 | { 'disk': None, # Disk object |
65 | 'mapped': False, # True if kpartx mapping exists | ||
66 | 'numpart': 0, # Number of allocate partitions | 60 | 'numpart': 0, # Number of allocate partitions |
67 | 'partitions': [], # Indexes to self.partitions | 61 | 'partitions': [], # Indexes to self.partitions |
68 | 'offset': 0, # Offset of next partition (in sectors) | 62 | 'offset': 0, # Offset of next partition (in sectors) |
@@ -98,40 +92,8 @@ class PartitionedMount(Mount): | |||
98 | # Converting MB to sectors for parted | 92 | # Converting MB to sectors for parted |
99 | size = size * 1024 * 1024 / self.sector_size | 93 | size = size * 1024 * 1024 / self.sector_size |
100 | 94 | ||
101 | # We need to handle subvolumes for btrfs | ||
102 | if fstype == "btrfs" and fsopts and fsopts.find("subvol=") != -1: | ||
103 | self.btrfscmd=find_binary_path("btrfs") | ||
104 | subvol = None | ||
105 | opts = fsopts.split(",") | ||
106 | for opt in opts: | ||
107 | if opt.find("subvol=") != -1: | ||
108 | subvol = opt.replace("subvol=", "").strip() | ||
109 | break | ||
110 | if not subvol: | ||
111 | raise MountError("No subvolume: %s" % fsopts) | ||
112 | self.subvolumes.append({'size': size, # In sectors | ||
113 | 'mountpoint': mountpoint, # Mount relative to chroot | ||
114 | 'fstype': fstype, # Filesystem type | ||
115 | 'fsopts': fsopts, # Filesystem mount options | ||
116 | 'disk_name': disk_name, # physical disk name holding partition | ||
117 | 'device': None, # kpartx device node for partition | ||
118 | 'mount': None, # Mount object | ||
119 | 'subvol': subvol, # Subvolume name | ||
120 | 'boot': boot, # Bootable flag | ||
121 | 'mounted': False # Mount flag | ||
122 | }) | ||
123 | |||
124 | # We still need partition for "/" or non-subvolume | 95 | # We still need partition for "/" or non-subvolume |
125 | if mountpoint == "/" or not fsopts or fsopts.find("subvol=") == -1: | 96 | if mountpoint == "/" or not fsopts: |
126 | # Don't need subvolume for "/" because it will be set as default subvolume | ||
127 | if fsopts and fsopts.find("subvol=") != -1: | ||
128 | opts = fsopts.split(",") | ||
129 | for opt in opts: | ||
130 | if opt.strip().startswith("subvol="): | ||
131 | opts.remove(opt) | ||
132 | break | ||
133 | fsopts = ",".join(opts) | ||
134 | |||
135 | part = { 'ks_pnum' : ks_pnum, # Partition number in the KS file | 97 | part = { 'ks_pnum' : ks_pnum, # Partition number in the KS file |
136 | 'size': size, # In sectors | 98 | 'size': size, # In sectors |
137 | 'mountpoint': mountpoint, # Mount relative to chroot | 99 | 'mountpoint': mountpoint, # Mount relative to chroot |
@@ -283,10 +245,6 @@ class PartitionedMount(Mount): | |||
283 | def __format_disks(self): | 245 | def __format_disks(self): |
284 | self.layout_partitions() | 246 | self.layout_partitions() |
285 | 247 | ||
286 | if self.skipformat: | ||
287 | msger.debug("Skipping disk format, because skipformat flag is set.") | ||
288 | return | ||
289 | |||
290 | for dev in self.disks.keys(): | 248 | for dev in self.disks.keys(): |
291 | d = self.disks[dev] | 249 | d = self.disks[dev] |
292 | msger.debug("Initializing partition table for %s" % \ | 250 | msger.debug("Initializing partition table for %s" % \ |
@@ -346,103 +304,6 @@ class PartitionedMount(Mount): | |||
346 | self.__run_parted(["-s", d['disk'].device, "set", | 304 | self.__run_parted(["-s", d['disk'].device, "set", |
347 | "%d" % p['num'], "lba", "off"]) | 305 | "%d" % p['num'], "lba", "off"]) |
348 | 306 | ||
349 | |||
350 | def __map_partitions(self): | ||
351 | """Load it if dm_snapshot isn't loaded. """ | ||
352 | load_module("dm_snapshot") | ||
353 | |||
354 | for dev in self.disks.keys(): | ||
355 | d = self.disks[dev] | ||
356 | if d['mapped']: | ||
357 | continue | ||
358 | |||
359 | msger.debug("Running kpartx on %s" % d['disk'].device ) | ||
360 | rc, kpartxOutput = runner.runtool([self.kpartx, "-l", "-v", d['disk'].device]) | ||
361 | kpartxOutput = kpartxOutput.splitlines() | ||
362 | |||
363 | if rc != 0: | ||
364 | raise MountError("Failed to query partition mapping for '%s'" % | ||
365 | d['disk'].device) | ||
366 | |||
367 | # Strip trailing blank and mask verbose output | ||
368 | i = 0 | ||
369 | while i < len(kpartxOutput) and kpartxOutput[i][0:4] != "loop": | ||
370 | i = i + 1 | ||
371 | kpartxOutput = kpartxOutput[i:] | ||
372 | |||
373 | # Make sure kpartx reported the right count of partitions | ||
374 | if len(kpartxOutput) != d['numpart']: | ||
375 | # If this disk has more than 3 partitions, then in case of MBR | ||
376 | # paritions there is an extended parition. Different versions | ||
377 | # of kpartx behave differently WRT the extended partition - | ||
378 | # some map it, some ignore it. This is why we do the below hack | ||
379 | # - if kpartx reported one more partition and the partition | ||
380 | # table type is "msdos" and the amount of partitions is more | ||
381 | # than 3, we just assume kpartx mapped the extended parition | ||
382 | # and we remove it. | ||
383 | if len(kpartxOutput) == d['numpart'] + 1 \ | ||
384 | and d['ptable_format'] == 'msdos' and len(kpartxOutput) > 3: | ||
385 | kpartxOutput.pop(3) | ||
386 | else: | ||
387 | raise MountError("Unexpected number of partitions from " \ | ||
388 | "kpartx: %d != %d" % \ | ||
389 | (len(kpartxOutput), d['numpart'])) | ||
390 | |||
391 | for i in range(len(kpartxOutput)): | ||
392 | line = kpartxOutput[i] | ||
393 | newdev = line.split()[0] | ||
394 | mapperdev = "/dev/mapper/" + newdev | ||
395 | loopdev = d['disk'].device + newdev[-1] | ||
396 | |||
397 | msger.debug("Dev %s: %s -> %s" % (newdev, loopdev, mapperdev)) | ||
398 | pnum = d['partitions'][i] | ||
399 | self.partitions[pnum]['device'] = loopdev | ||
400 | |||
401 | # grub's install wants partitions to be named | ||
402 | # to match their parent device + partition num | ||
403 | # kpartx doesn't work like this, so we add compat | ||
404 | # symlinks to point to /dev/mapper | ||
405 | if os.path.lexists(loopdev): | ||
406 | os.unlink(loopdev) | ||
407 | os.symlink(mapperdev, loopdev) | ||
408 | |||
409 | msger.debug("Adding partx mapping for %s" % d['disk'].device) | ||
410 | rc = runner.show([self.kpartx, "-v", "-a", d['disk'].device]) | ||
411 | |||
412 | if rc != 0: | ||
413 | # Make sure that the device maps are also removed on error case. | ||
414 | # The d['mapped'] isn't set to True if the kpartx fails so | ||
415 | # failed mapping will not be cleaned on cleanup either. | ||
416 | runner.quiet([self.kpartx, "-d", d['disk'].device]) | ||
417 | raise MountError("Failed to map partitions for '%s'" % | ||
418 | d['disk'].device) | ||
419 | |||
420 | # FIXME: there is a bit delay for multipath device setup, | ||
421 | # wait 10ms for the setup | ||
422 | import time | ||
423 | time.sleep(10) | ||
424 | d['mapped'] = True | ||
425 | |||
426 | def __unmap_partitions(self): | ||
427 | for dev in self.disks.keys(): | ||
428 | d = self.disks[dev] | ||
429 | if not d['mapped']: | ||
430 | continue | ||
431 | |||
432 | msger.debug("Removing compat symlinks") | ||
433 | for pnum in d['partitions']: | ||
434 | if self.partitions[pnum]['device'] != None: | ||
435 | os.unlink(self.partitions[pnum]['device']) | ||
436 | self.partitions[pnum]['device'] = None | ||
437 | |||
438 | msger.debug("Unmapping %s" % d['disk'].device) | ||
439 | rc = runner.quiet([self.kpartx, "-d", d['disk'].device]) | ||
440 | if rc != 0: | ||
441 | raise MountError("Failed to unmap partitions for '%s'" % | ||
442 | d['disk'].device) | ||
443 | |||
444 | d['mapped'] = False | ||
445 | |||
446 | def __calculate_mountorder(self): | 307 | def __calculate_mountorder(self): |
447 | msger.debug("Calculating mount order") | 308 | msger.debug("Calculating mount order") |
448 | for p in self.partitions: | 309 | for p in self.partitions: |
@@ -457,7 +318,6 @@ class PartitionedMount(Mount): | |||
457 | def cleanup(self): | 318 | def cleanup(self): |
458 | Mount.cleanup(self) | 319 | Mount.cleanup(self) |
459 | if self.disks: | 320 | if self.disks: |
460 | self.__unmap_partitions() | ||
461 | for dev in self.disks.keys(): | 321 | for dev in self.disks.keys(): |
462 | d = self.disks[dev] | 322 | d = self.disks[dev] |
463 | try: | 323 | try: |
@@ -466,7 +326,6 @@ class PartitionedMount(Mount): | |||
466 | pass | 326 | pass |
467 | 327 | ||
468 | def unmount(self): | 328 | def unmount(self): |
469 | self.__unmount_subvolumes() | ||
470 | for mp in self.unmountOrder: | 329 | for mp in self.unmountOrder: |
471 | if mp == 'swap': | 330 | if mp == 'swap': |
472 | continue | 331 | continue |
@@ -478,217 +337,11 @@ class PartitionedMount(Mount): | |||
478 | 337 | ||
479 | if p['mount'] != None: | 338 | if p['mount'] != None: |
480 | try: | 339 | try: |
481 | # Create subvolume snapshot here | ||
482 | if p['fstype'] == "btrfs" and p['mountpoint'] == "/" and not self.snapshot_created: | ||
483 | self.__create_subvolume_snapshots(p, p["mount"]) | ||
484 | p['mount'].cleanup() | 340 | p['mount'].cleanup() |
485 | except: | 341 | except: |
486 | pass | 342 | pass |
487 | p['mount'] = None | 343 | p['mount'] = None |
488 | 344 | ||
489 | # Only for btrfs | ||
490 | def __get_subvolume_id(self, rootpath, subvol): | ||
491 | if not self.btrfscmd: | ||
492 | self.btrfscmd=find_binary_path("btrfs") | ||
493 | argv = [ self.btrfscmd, "subvolume", "list", rootpath ] | ||
494 | |||
495 | rc, out = runner.runtool(argv) | ||
496 | msger.debug(out) | ||
497 | |||
498 | if rc != 0: | ||
499 | raise MountError("Failed to get subvolume id from %s', return code: %d." % (rootpath, rc)) | ||
500 | |||
501 | subvolid = -1 | ||
502 | for line in out.splitlines(): | ||
503 | if line.endswith(" path %s" % subvol): | ||
504 | subvolid = line.split()[1] | ||
505 | if not subvolid.isdigit(): | ||
506 | raise MountError("Invalid subvolume id: %s" % subvolid) | ||
507 | subvolid = int(subvolid) | ||
508 | break | ||
509 | return subvolid | ||
510 | |||
511 | def __create_subvolume_metadata(self, p, pdisk): | ||
512 | if len(self.subvolumes) == 0: | ||
513 | return | ||
514 | |||
515 | argv = [ self.btrfscmd, "subvolume", "list", pdisk.mountdir ] | ||
516 | rc, out = runner.runtool(argv) | ||
517 | msger.debug(out) | ||
518 | |||
519 | if rc != 0: | ||
520 | raise MountError("Failed to get subvolume id from %s', return code: %d." % (pdisk.mountdir, rc)) | ||
521 | |||
522 | subvolid_items = out.splitlines() | ||
523 | subvolume_metadata = "" | ||
524 | for subvol in self.subvolumes: | ||
525 | for line in subvolid_items: | ||
526 | if line.endswith(" path %s" % subvol["subvol"]): | ||
527 | subvolid = line.split()[1] | ||
528 | if not subvolid.isdigit(): | ||
529 | raise MountError("Invalid subvolume id: %s" % subvolid) | ||
530 | |||
531 | subvolid = int(subvolid) | ||
532 | opts = subvol["fsopts"].split(",") | ||
533 | for opt in opts: | ||
534 | if opt.strip().startswith("subvol="): | ||
535 | opts.remove(opt) | ||
536 | break | ||
537 | fsopts = ",".join(opts) | ||
538 | subvolume_metadata += "%d\t%s\t%s\t%s\n" % (subvolid, subvol["subvol"], subvol['mountpoint'], fsopts) | ||
539 | |||
540 | if subvolume_metadata: | ||
541 | fd = open("%s/.subvolume_metadata" % pdisk.mountdir, "w") | ||
542 | fd.write(subvolume_metadata) | ||
543 | fd.close() | ||
544 | |||
545 | def __get_subvolume_metadata(self, p, pdisk): | ||
546 | subvolume_metadata_file = "%s/.subvolume_metadata" % pdisk.mountdir | ||
547 | if not os.path.exists(subvolume_metadata_file): | ||
548 | return | ||
549 | |||
550 | fd = open(subvolume_metadata_file, "r") | ||
551 | content = fd.read() | ||
552 | fd.close() | ||
553 | |||
554 | for line in content.splitlines(): | ||
555 | items = line.split("\t") | ||
556 | if items and len(items) == 4: | ||
557 | self.subvolumes.append({'size': 0, # In sectors | ||
558 | 'mountpoint': items[2], # Mount relative to chroot | ||
559 | 'fstype': "btrfs", # Filesystem type | ||
560 | 'fsopts': items[3] + ",subvol=%s" % items[1], # Filesystem mount options | ||
561 | 'disk_name': p['disk_name'], # physical disk name holding partition | ||
562 | 'device': None, # kpartx device node for partition | ||
563 | 'mount': None, # Mount object | ||
564 | 'subvol': items[1], # Subvolume name | ||
565 | 'boot': False, # Bootable flag | ||
566 | 'mounted': False # Mount flag | ||
567 | }) | ||
568 | |||
569 | def __create_subvolumes(self, p, pdisk): | ||
570 | """ Create all the subvolumes. """ | ||
571 | |||
572 | for subvol in self.subvolumes: | ||
573 | argv = [ self.btrfscmd, "subvolume", "create", pdisk.mountdir + "/" + subvol["subvol"]] | ||
574 | |||
575 | rc = runner.show(argv) | ||
576 | if rc != 0: | ||
577 | raise MountError("Failed to create subvolume '%s', return code: %d." % (subvol["subvol"], rc)) | ||
578 | |||
579 | # Set default subvolume, subvolume for "/" is default | ||
580 | subvol = None | ||
581 | for subvolume in self.subvolumes: | ||
582 | if subvolume["mountpoint"] == "/" and p['disk_name'] == subvolume['disk_name']: | ||
583 | subvol = subvolume | ||
584 | break | ||
585 | |||
586 | if subvol: | ||
587 | # Get default subvolume id | ||
588 | subvolid = self. __get_subvolume_id(pdisk.mountdir, subvol["subvol"]) | ||
589 | # Set default subvolume | ||
590 | if subvolid != -1: | ||
591 | rc = runner.show([ self.btrfscmd, "subvolume", "set-default", "%d" % subvolid, pdisk.mountdir]) | ||
592 | if rc != 0: | ||
593 | raise MountError("Failed to set default subvolume id: %d', return code: %d." % (subvolid, rc)) | ||
594 | |||
595 | self.__create_subvolume_metadata(p, pdisk) | ||
596 | |||
597 | def __mount_subvolumes(self, p, pdisk): | ||
598 | if self.skipformat: | ||
599 | # Get subvolume info | ||
600 | self.__get_subvolume_metadata(p, pdisk) | ||
601 | # Set default mount options | ||
602 | if len(self.subvolumes) != 0: | ||
603 | for subvol in self.subvolumes: | ||
604 | if subvol["mountpoint"] == p["mountpoint"] == "/": | ||
605 | opts = subvol["fsopts"].split(",") | ||
606 | for opt in opts: | ||
607 | if opt.strip().startswith("subvol="): | ||
608 | opts.remove(opt) | ||
609 | break | ||
610 | pdisk.fsopts = ",".join(opts) | ||
611 | break | ||
612 | |||
613 | if len(self.subvolumes) == 0: | ||
614 | # Return directly if no subvolumes | ||
615 | return | ||
616 | |||
617 | # Remount to make default subvolume mounted | ||
618 | rc = runner.show([self.umountcmd, pdisk.mountdir]) | ||
619 | if rc != 0: | ||
620 | raise MountError("Failed to umount %s" % pdisk.mountdir) | ||
621 | |||
622 | rc = runner.show([self.mountcmd, "-o", pdisk.fsopts, pdisk.disk.device, pdisk.mountdir]) | ||
623 | if rc != 0: | ||
624 | raise MountError("Failed to umount %s" % pdisk.mountdir) | ||
625 | |||
626 | for subvol in self.subvolumes: | ||
627 | if subvol["mountpoint"] == "/": | ||
628 | continue | ||
629 | subvolid = self. __get_subvolume_id(pdisk.mountdir, subvol["subvol"]) | ||
630 | if subvolid == -1: | ||
631 | msger.debug("WARNING: invalid subvolume %s" % subvol["subvol"]) | ||
632 | continue | ||
633 | # Replace subvolume name with subvolume ID | ||
634 | opts = subvol["fsopts"].split(",") | ||
635 | for opt in opts: | ||
636 | if opt.strip().startswith("subvol="): | ||
637 | opts.remove(opt) | ||
638 | break | ||
639 | |||
640 | opts.extend(["subvolrootid=0", "subvol=%s" % subvol["subvol"]]) | ||
641 | fsopts = ",".join(opts) | ||
642 | subvol['fsopts'] = fsopts | ||
643 | mountpoint = self.mountdir + subvol['mountpoint'] | ||
644 | makedirs(mountpoint) | ||
645 | rc = runner.show([self.mountcmd, "-o", fsopts, pdisk.disk.device, mountpoint]) | ||
646 | if rc != 0: | ||
647 | raise MountError("Failed to mount subvolume %s to %s" % (subvol["subvol"], mountpoint)) | ||
648 | subvol["mounted"] = True | ||
649 | |||
650 | def __unmount_subvolumes(self): | ||
651 | """ It may be called multiple times, so we need to chekc if it is still mounted. """ | ||
652 | for subvol in self.subvolumes: | ||
653 | if subvol["mountpoint"] == "/": | ||
654 | continue | ||
655 | if not subvol["mounted"]: | ||
656 | continue | ||
657 | mountpoint = self.mountdir + subvol['mountpoint'] | ||
658 | rc = runner.show([self.umountcmd, mountpoint]) | ||
659 | if rc != 0: | ||
660 | raise MountError("Failed to unmount subvolume %s from %s" % (subvol["subvol"], mountpoint)) | ||
661 | subvol["mounted"] = False | ||
662 | |||
663 | def __create_subvolume_snapshots(self, p, pdisk): | ||
664 | import time | ||
665 | |||
666 | if self.snapshot_created: | ||
667 | return | ||
668 | |||
669 | # Remount with subvolid=0 | ||
670 | rc = runner.show([self.umountcmd, pdisk.mountdir]) | ||
671 | if rc != 0: | ||
672 | raise MountError("Failed to umount %s" % pdisk.mountdir) | ||
673 | if pdisk.fsopts: | ||
674 | mountopts = pdisk.fsopts + ",subvolid=0" | ||
675 | else: | ||
676 | mountopts = "subvolid=0" | ||
677 | rc = runner.show([self.mountcmd, "-o", mountopts, pdisk.disk.device, pdisk.mountdir]) | ||
678 | if rc != 0: | ||
679 | raise MountError("Failed to umount %s" % pdisk.mountdir) | ||
680 | |||
681 | # Create all the subvolume snapshots | ||
682 | snapshotts = time.strftime("%Y%m%d-%H%M") | ||
683 | for subvol in self.subvolumes: | ||
684 | subvolpath = pdisk.mountdir + "/" + subvol["subvol"] | ||
685 | snapshotpath = subvolpath + "_%s-1" % snapshotts | ||
686 | rc = runner.show([ self.btrfscmd, "subvolume", "snapshot", subvolpath, snapshotpath ]) | ||
687 | if rc != 0: | ||
688 | raise MountError("Failed to create subvolume snapshot '%s' for '%s', return code: %d." % (snapshotpath, subvolpath, rc)) | ||
689 | |||
690 | self.snapshot_created = True | ||
691 | |||
692 | def __install_partition(self, num, source_file, start, size): | 345 | def __install_partition(self, num, source_file, start, size): |
693 | """ | 346 | """ |
694 | Install source_file contents into a partition. | 347 | Install source_file contents into a partition. |
@@ -734,7 +387,3 @@ class PartitionedMount(Mount): | |||
734 | self.__calculate_mountorder() | 387 | self.__calculate_mountorder() |
735 | 388 | ||
736 | return | 389 | return |
737 | |||
738 | def resparse(self, size = None): | ||
739 | # Can't re-sparse a disk image - too hard | ||
740 | pass | ||