16 files changed, 7410 insertions, 0 deletions

diff --git a/scripts/lib/mic/utils/BmapCreate.py b/scripts/lib/mic/utils/BmapCreate.py
new file mode 100644
index 0000000000..65b19a5f46
--- /dev/null
+++ b/scripts/lib/mic/utils/BmapCreate.py
@@ -0,0 +1,298 @@
+""" This module implements the block map (bmap) creation functionality and
+provides the corresponding API in form of the 'BmapCreate' class.
+
+The idea is that while images files may generally be very large (e.g., 4GiB),
+they may nevertheless contain only little real data, e.g., 512MiB. This data
+are files, directories, file-system meta-data, partition table, etc. When
+copying the image to the target device, you do not have to copy all the 4GiB of
+data, you can copy only 512MiB of it, which is 4 times less, so copying should
+presumably be 4 times faster.
+
+The block map file is an XML file which contains a list of blocks which have to
+be copied to the target device. The other blocks are not used and there is no
+need to copy them. The XML file also contains some additional information like
+block size, image size, count of mapped blocks, etc. There are also many
+commentaries, so it is human-readable.
+
+The image has to be a sparse file. Generally, this means that when you generate
+this image file, you should start with a huge sparse file which contains a
+single hole spanning the entire file. Then you should partition it, write all
+the data (probably by means of loop-back mounting the image or parts of it),
+etc. The end result should be a sparse file where mapped areas represent useful
+parts of the image and holes represent useless parts of the image, which do not
+have to be copied when copying the image to the target device.
+
+This module uses the FIBMAP ioctl to detect holes. """
+
+# Disable the following pylint recommendations:
+#   *  Too many instance attributes - R0902
+#   *  Too few public methods - R0903
+# pylint: disable=R0902,R0903
+
+import hashlib
+from mic.utils.misc import human_size
+from mic.utils import Fiemap
+
+# The bmap format version we generate
+SUPPORTED_BMAP_VERSION = "1.3"
+
+_BMAP_START_TEMPLATE = \
+"""<?xml version="1.0" ?>
+<!-- This file contains the block map for an image file, which is basically
+     a list of useful (mapped) block numbers in the image file. In other words,
+     it lists only those blocks which contain data (boot sector, partition
+     table, file-system metadata, files, directories, extents, etc). These
+     blocks have to be copied to the target device. The other blocks do not
+     contain any useful data and do not have to be copied to the target
+     device.
+
+     The block map an optimization which allows to copy or flash the image to
+     the image quicker than copying of flashing the entire image. This is
+     because with bmap less data is copied: <MappedBlocksCount> blocks instead
+     of <BlocksCount> blocks.
+
+     Besides the machine-readable data, this file contains useful commentaries
+     which contain human-readable information like image size, percentage of
+     mapped data, etc.
+
+     The 'version' attribute is the block map file format version in the
+     'major.minor' format. The version major number is increased whenever an
+     incompatible block map format change is made. The minor number changes
+     in case of minor backward-compatible changes. -->
+
+<bmap version="%s">
+    <!-- Image size in bytes: %s -->
+    <ImageSize> %u </ImageSize>
+
+    <!-- Size of a block in bytes -->
+    <BlockSize> %u </BlockSize>
+
+    <!-- Count of blocks in the image file -->
+    <BlocksCount> %u </BlocksCount>
+
+"""
+
+class Error(Exception):
+    """ A class for exceptions generated by this module. We currently support
+    only one type of exceptions, and we basically throw human-readable problem
+    description in case of errors. """
+    pass
+
+class BmapCreate:
+    """ This class implements the bmap creation functionality. To generate a
+    bmap for an image (which is supposedly a sparse file), you should first
+    create an instance of 'BmapCreate' and provide:
+
+    * full path or a file-like object of the image to create bmap for
+    * full path or a file object to use for writing the results to
+
+    Then you should invoke the 'generate()' method of this class. It will use
+    the FIEMAP ioctl to generate the bmap. """
+
+    def _open_image_file(self):
+        """ Open the image file. """
+
+        try:
+            self._f_image = open(self._image_path, 'rb')
+        except IOError as err:
+            raise Error("cannot open image file '%s': %s" \
+                        % (self._image_path, err))
+
+        self._f_image_needs_close = True
+
+    def _open_bmap_file(self):
+        """ Open the bmap file. """
+
+        try:
+            self._f_bmap = open(self._bmap_path, 'w+')
+        except IOError as err:
+            raise Error("cannot open bmap file '%s': %s" \
+                        % (self._bmap_path, err))
+
+        self._f_bmap_needs_close = True
+
+    def __init__(self, image, bmap):
+        """ Initialize a class instance:
+        * image - full path or a file-like object of the image to create bmap
+                  for
+        * bmap  - full path or a file object to use for writing the resulting
+                  bmap to """
+
+        self.image_size = None
+        self.image_size_human = None
+        self.block_size = None
+        self.blocks_cnt = None
+        self.mapped_cnt = None
+        self.mapped_size = None
+        self.mapped_size_human = None
+        self.mapped_percent = None
+
+        self._mapped_count_pos1 = None
+        self._mapped_count_pos2 = None
+        self._sha1_pos = None
+
+        self._f_image_needs_close = False
+        self._f_bmap_needs_close = False
+
+        if hasattr(image, "read"):
+            self._f_image = image
+            self._image_path = image.name
+        else:
+            self._image_path = image
+            self._open_image_file()
+
+        if hasattr(bmap, "read"):
+            self._f_bmap = bmap
+            self._bmap_path = bmap.name
+        else:
+            self._bmap_path = bmap
+            self._open_bmap_file()
+
+        self.fiemap = Fiemap.Fiemap(self._f_image)
+
+        self.image_size = self.fiemap.image_size
+        self.image_size_human = human_size(self.image_size)
+        if self.image_size == 0:
+            raise Error("cannot generate bmap for zero-sized image file '%s'" \
+                        % self._image_path)
+
+        self.block_size = self.fiemap.block_size
+        self.blocks_cnt = self.fiemap.blocks_cnt
+
+    def _bmap_file_start(self):
+        """ A helper function which generates the starting contents of the
+        block map file: the header comment, image size, block size, etc. """
+
+        # We do not know the amount of mapped blocks at the moment, so just put
+        # whitespaces instead of real numbers. Assume the longest possible
+        # numbers.
+        mapped_count = ' ' * len(str(self.image_size))
+        mapped_size_human = ' ' * len(self.image_size_human)
+
+        xml = _BMAP_START_TEMPLATE \
+               % (SUPPORTED_BMAP_VERSION, self.image_size_human,
+                  self.image_size, self.block_size, self.blocks_cnt)
+        xml += "    <!-- Count of mapped blocks: "
+
+        self._f_bmap.write(xml)
+        self._mapped_count_pos1 = self._f_bmap.tell()
+
+        # Just put white-spaces instead of real information about mapped blocks
+        xml  = "%s or %.1f    -->\n" % (mapped_size_human, 100.0)
+        xml += "    <MappedBlocksCount> "
+
+        self._f_bmap.write(xml)
+        self._mapped_count_pos2 = self._f_bmap.tell()
+
+        xml  = "%s </MappedBlocksCount>\n\n" % mapped_count
+
+        # pylint: disable=C0301
+        xml += "    <!-- The checksum of this bmap file. When it is calculated, the value of\n"
+        xml += "         the SHA1 checksum has be zeoro (40 ASCII \"0\" symbols). -->\n"
+        xml += "    <BmapFileSHA1> "
+
+        self._f_bmap.write(xml)
+        self._sha1_pos = self._f_bmap.tell()
+
+        xml = "0" * 40 + " </BmapFileSHA1>\n\n"
+        xml += "    <!-- The block map which consists of elements which may either be a\n"
+        xml += "         range of blocks or a single block. The 'sha1' attribute (if present)\n"
+        xml += "         is the SHA1 checksum of this blocks range. -->\n"
+        xml += "    <BlockMap>\n"
+        # pylint: enable=C0301
+
+        self._f_bmap.write(xml)
+
+    def _bmap_file_end(self):
+        """ A helper function which generates the final parts of the block map
+        file: the ending tags and the information about the amount of mapped
+        blocks. """
+
+        xml =  "    </BlockMap>\n"
+        xml += "</bmap>\n"
+
+        self._f_bmap.write(xml)
+
+        self._f_bmap.seek(self._mapped_count_pos1)
+        self._f_bmap.write("%s or %.1f%%" % \
+                           (self.mapped_size_human, self.mapped_percent))
+
+        self._f_bmap.seek(self._mapped_count_pos2)
+        self._f_bmap.write("%u" % self.mapped_cnt)
+
+        self._f_bmap.seek(0)
+        sha1 = hashlib.sha1(self._f_bmap.read()).hexdigest()
+        self._f_bmap.seek(self._sha1_pos)
+        self._f_bmap.write("%s" % sha1)
+
+    def _calculate_sha1(self, first, last):
+        """ A helper function which calculates SHA1 checksum for the range of
+        blocks of the image file: from block 'first' to block 'last'. """
+
+        start = first * self.block_size
+        end = (last + 1) * self.block_size
+
+        self._f_image.seek(start)
+        hash_obj = hashlib.new("sha1")
+
+        chunk_size = 1024*1024
+        to_read = end - start
+        read = 0
+
+        while read < to_read:
+            if read + chunk_size > to_read:
+                chunk_size = to_read - read
+            chunk = self._f_image.read(chunk_size)
+            hash_obj.update(chunk)
+            read += chunk_size
+
+        return hash_obj.hexdigest()
+
+    def generate(self, include_checksums = True):
+        """ Generate bmap for the image file. If 'include_checksums' is 'True',
+        also generate SHA1 checksums for block ranges. """
+
+        # Save image file position in order to restore it at the end
+        image_pos = self._f_image.tell()
+
+        self._bmap_file_start()
+
+        # Generate the block map and write it to the XML block map
+        # file as we go.
+        self.mapped_cnt = 0
+        for first, last in self.fiemap.get_mapped_ranges(0, self.blocks_cnt):
+            self.mapped_cnt += last - first + 1
+            if include_checksums:
+                sha1 = self._calculate_sha1(first, last)
+                sha1 = " sha1=\"%s\"" % sha1
+            else:
+                sha1 = ""
+
+            if first != last:
+                self._f_bmap.write("        <Range%s> %s-%s </Range>\n" \
+                                   % (sha1, first, last))
+            else:
+                self._f_bmap.write("        <Range%s> %s </Range>\n" \
+                                   % (sha1, first))
+
+        self.mapped_size = self.mapped_cnt * self.block_size
+        self.mapped_size_human = human_size(self.mapped_size)
+        self.mapped_percent = (self.mapped_cnt * 100.0) /  self.blocks_cnt
+
+        self._bmap_file_end()
+
+        try:
+            self._f_bmap.flush()
+        except IOError as err:
+            raise Error("cannot flush the bmap file '%s': %s" \
+                        % (self._bmap_path, err))
+
+        self._f_image.seek(image_pos)
+
+    def __del__(self):
+        """ The class destructor which closes the opened files. """
+
+        if self._f_image_needs_close:
+            self._f_image.close()
+        if self._f_bmap_needs_close:
+            self._f_bmap.close()
diff --git a/scripts/lib/mic/utils/Fiemap.py b/scripts/lib/mic/utils/Fiemap.py
new file mode 100644
index 0000000000..f2db6ff0b8
--- /dev/null
+++ b/scripts/lib/mic/utils/Fiemap.py
@@ -0,0 +1,252 @@
+""" This module implements python API for the FIEMAP ioctl. The FIEMAP ioctl
+allows to find holes and mapped areas in a file. """
+
+# Note, a lot of code in this module is not very readable, because it deals
+# with the rather complex FIEMAP ioctl. To understand the code, you need to
+# know the FIEMAP interface, which is documented in the
+# Documentation/filesystems/fiemap.txt file in the Linux kernel sources.
+
+# Disable the following pylint recommendations:
+#   * Too many instance attributes (R0902)
+# pylint: disable=R0902
+
+import os
+import struct
+import array
+import fcntl
+from mic.utils.misc import get_block_size
+
+# Format string for 'struct fiemap'
+_FIEMAP_FORMAT = "=QQLLLL"
+# sizeof(struct fiemap)
+_FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT)
+# Format string for 'struct fiemap_extent'
+_FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL"
+# sizeof(struct fiemap_extent)
+_FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT)
+# The FIEMAP ioctl number
+_FIEMAP_IOCTL = 0xC020660B
+
+# Minimum buffer which is required for 'class Fiemap' to operate
+MIN_BUFFER_SIZE = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE
+# The default buffer size for 'class Fiemap'
+DEFAULT_BUFFER_SIZE = 256 * 1024
+
+class Error(Exception):
+    """ A class for exceptions generated by this module. We currently support
+    only one type of exceptions, and we basically throw human-readable problem
+    description in case of errors. """
+    pass
+
+class Fiemap:
+    """ This class provides API to the FIEMAP ioctl. Namely, it allows to
+    iterate over all mapped blocks and over all holes. """
+
+    def _open_image_file(self):
+        """ Open the image file. """
+
+        try:
+            self._f_image = open(self._image_path, 'rb')
+        except IOError as err:
+            raise Error("cannot open image file '%s': %s" \
+                        % (self._image_path, err))
+
+        self._f_image_needs_close = True
+
+    def __init__(self, image, buf_size = DEFAULT_BUFFER_SIZE):
+        """ Initialize a class instance. The 'image' argument is full path to
+        the file to operate on, or a file object to operate on.
+
+        The 'buf_size' argument is the size of the buffer for 'struct
+        fiemap_extent' elements which will be used when invoking the FIEMAP
+        ioctl. The larger is the buffer, the less times the FIEMAP ioctl will
+        be invoked. """
+
+        self._f_image_needs_close = False
+
+        if hasattr(image, "fileno"):
+            self._f_image = image
+            self._image_path = image.name
+        else:
+            self._image_path = image
+            self._open_image_file()
+
+        # Validate 'buf_size'
+        if buf_size < MIN_BUFFER_SIZE:
+            raise Error("too small buffer (%d bytes), minimum is %d bytes" \
+                    % (buf_size, MIN_BUFFER_SIZE))
+
+        # How many 'struct fiemap_extent' elements fit the buffer
+        buf_size -= _FIEMAP_SIZE
+        self._fiemap_extent_cnt = buf_size / _FIEMAP_EXTENT_SIZE
+        self._buf_size = self._fiemap_extent_cnt * _FIEMAP_EXTENT_SIZE
+        self._buf_size += _FIEMAP_SIZE
+
+        # Allocate a mutable buffer for the FIEMAP ioctl
+        self._buf = array.array('B', [0] * self._buf_size)
+
+        self.image_size = os.fstat(self._f_image.fileno()).st_size
+
+        try:
+            self.block_size = get_block_size(self._f_image)
+        except IOError as err:
+            raise Error("cannot get block size for '%s': %s" \
+                        % (self._image_path, err))
+
+        self.blocks_cnt = self.image_size + self.block_size - 1
+        self.blocks_cnt /= self.block_size
+
+        # Synchronize the image file to make sure FIEMAP returns correct values
+        try:
+            self._f_image.flush()
+        except IOError as err:
+            raise Error("cannot flush image file '%s': %s" \
+                        % (self._image_path, err))
+        try:
+            os.fsync(self._f_image.fileno()),
+        except OSError as err:
+            raise Error("cannot synchronize image file '%s': %s " \
+                        % (self._image_path, err.strerror))
+
+        # Check if the FIEMAP ioctl is supported
+        self.block_is_mapped(0)
+
+    def __del__(self):
+        """ The class destructor which closes the opened files. """
+
+        if self._f_image_needs_close:
+            self._f_image.close()
+
+    def _invoke_fiemap(self, block, count):
+        """ Invoke the FIEMAP ioctl for 'count' blocks of the file starting from
+        block number 'block'.
+
+        The full result of the operation is stored in 'self._buf' on exit.
+        Returns the unpacked 'struct fiemap' data structure in form of a python
+        list (just like 'struct.upack()'). """
+
+        if block < 0 or block >= self.blocks_cnt:
+            raise Error("bad block number %d, should be within [0, %d]" \
+                        % (block, self.blocks_cnt))
+
+        # Initialize the 'struct fiemap' part of the buffer
+        struct.pack_into(_FIEMAP_FORMAT, self._buf, 0, block * self.block_size,
+                         count * self.block_size, 0, 0,
+                         self._fiemap_extent_cnt, 0)
+
+        try:
+            fcntl.ioctl(self._f_image, _FIEMAP_IOCTL, self._buf, 1)
+        except IOError as err:
+            error_msg = "the FIEMAP ioctl failed for '%s': %s" \
+                        % (self._image_path, err)
+            if err.errno == os.errno.EPERM or err.errno == os.errno.EACCES:
+                # The FIEMAP ioctl was added in kernel version 2.6.28 in 2008
+                error_msg += " (looks like your kernel does not support FIEMAP)"
+
+            raise Error(error_msg)
+
+        return struct.unpack(_FIEMAP_FORMAT, self._buf[:_FIEMAP_SIZE])
+
+    def block_is_mapped(self, block):
+        """ This function returns 'True' if block number 'block' of the image
+        file is mapped and 'False' otherwise. """
+
+        struct_fiemap = self._invoke_fiemap(block, 1)
+
+        # The 3rd element of 'struct_fiemap' is the 'fm_mapped_extents' field.
+        # If it contains zero, the block is not mapped, otherwise it is
+        # mapped.
+        return bool(struct_fiemap[3])
+
+    def block_is_unmapped(self, block):
+        """ This function returns 'True' if block number 'block' of the image
+        file is not mapped (hole) and 'False' otherwise. """
+
+        return not self.block_is_mapped(block)
+
+    def _unpack_fiemap_extent(self, index):
+        """ Unpack a 'struct fiemap_extent' structure object number 'index'
+        from the internal 'self._buf' buffer. """
+
+        offset = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE * index
+        return struct.unpack(_FIEMAP_EXTENT_FORMAT,
+                             self._buf[offset : offset + _FIEMAP_EXTENT_SIZE])
+
+    def _do_get_mapped_ranges(self, start, count):
+        """ Implements most the functionality for the  'get_mapped_ranges()'
+        generator: invokes the FIEMAP ioctl, walks through the mapped
+        extents and yields mapped block ranges. However, the ranges may be
+        consecutive (e.g., (1, 100), (100, 200)) and 'get_mapped_ranges()'
+        simply merges them. """
+
+        block = start
+        while block < start + count:
+            struct_fiemap = self._invoke_fiemap(block, count)
+
+            mapped_extents = struct_fiemap[3]
+            if mapped_extents == 0:
+                # No more mapped blocks
+                return
+
+            extent = 0
+            while extent < mapped_extents:
+                fiemap_extent = self._unpack_fiemap_extent(extent)
+
+                # Start of the extent
+                extent_start = fiemap_extent[0]
+                # Starting block number of the extent
+                extent_block = extent_start / self.block_size
+                # Length of the extent
+                extent_len = fiemap_extent[2]
+                # Count of blocks in the extent
+                extent_count = extent_len / self.block_size
+
+                # Extent length and offset have to be block-aligned
+                assert extent_start % self.block_size == 0
+                assert extent_len % self.block_size == 0
+
+                if extent_block > start + count - 1:
+                    return
+
+                first = max(extent_block, block)
+                last = min(extent_block + extent_count, start + count) - 1
+                yield (first, last)
+
+                extent += 1
+
+            block = extent_block + extent_count
+
+    def get_mapped_ranges(self, start, count):
+        """ A generator which yields ranges of mapped blocks in the file. The
+        ranges are tuples of 2 elements: [first, last], where 'first' is the
+        first mapped block and 'last' is the last mapped block.
+
+        The ranges are yielded for the area of the file of size 'count' blocks,
+        starting from block 'start'. """
+
+        iterator = self._do_get_mapped_ranges(start, count)
+
+        first_prev, last_prev = iterator.next()
+
+        for first, last in iterator:
+            if last_prev == first - 1:
+                last_prev = last
+            else:
+                yield (first_prev, last_prev)
+                first_prev, last_prev = first, last
+
+        yield (first_prev, last_prev)
+
+    def get_unmapped_ranges(self, start, count):
+        """ Just like 'get_mapped_ranges()', but yields unmapped block ranges
+        instead (holes). """
+
+        hole_first = start
+        for first, last in self._do_get_mapped_ranges(start, count):
+            if first > hole_first:
+                yield (hole_first, first - 1)
+
+            hole_first = last + 1
+
+        if hole_first < start + count:
+            yield (hole_first, start + count - 1)
diff --git a/scripts/lib/mic/utils/__init__.py b/scripts/lib/mic/utils/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/scripts/lib/mic/utils/__init__.py
diff --git a/scripts/lib/mic/utils/cmdln.py b/scripts/lib/mic/utils/cmdln.py
new file mode 100644
index 0000000000..b099473ee4
--- /dev/null
+++ b/scripts/lib/mic/utils/cmdln.py
@@ -0,0 +1,1586 @@
+#!/usr/bin/env python
+# Copyright (c) 2002-2007 ActiveState Software Inc.
+# License: MIT (see LICENSE.txt for license details)
+# Author:  Trent Mick
+# Home:    http://trentm.com/projects/cmdln/
+
+"""An improvement on Python's standard cmd.py module.
+
+As with cmd.py, this module provides "a simple framework for writing
+line-oriented command intepreters."  This module provides a 'RawCmdln'
+class that fixes some design flaws in cmd.Cmd, making it more scalable
+and nicer to use for good 'cvs'- or 'svn'-style command line interfaces
+or simple shells.  And it provides a 'Cmdln' class that add
+optparse-based option processing. Basically you use it like this:
+
+    import cmdln
+
+    class MySVN(cmdln.Cmdln):
+        name = "svn"
+
+        @cmdln.alias('stat', 'st')
+        @cmdln.option('-v', '--verbose', action='store_true'
+                      help='print verbose information')
+        def do_status(self, subcmd, opts, *paths):
+            print "handle 'svn status' command"
+
+        #...
+
+    if __name__ == "__main__":
+        shell = MySVN()
+        retval = shell.main()
+        sys.exit(retval)
+
+See the README.txt or <http://trentm.com/projects/cmdln/> for more
+details.
+"""
+
+__version_info__ = (1, 1, 2)
+__version__ = '.'.join(map(str, __version_info__))
+
+import os
+import sys
+import re
+import cmd
+import optparse
+from pprint import pprint
+import sys
+
+
+
+
+#---- globals
+
+LOOP_ALWAYS, LOOP_NEVER, LOOP_IF_EMPTY = range(3)
+
+# An unspecified optional argument when None is a meaningful value.
+_NOT_SPECIFIED = ("Not", "Specified")
+
+# Pattern to match a TypeError message from a call that
+# failed because of incorrect number of arguments (see
+# Python/getargs.c).
+_INCORRECT_NUM_ARGS_RE = re.compile(
+    r"(takes [\w ]+ )(\d+)( arguments? \()(\d+)( given\))")
+
+
+
+#---- exceptions
+
+class CmdlnError(Exception):
+    """A cmdln.py usage error."""
+    def __init__(self, msg):
+        self.msg = msg
+    def __str__(self):
+        return self.msg
+
+class CmdlnUserError(Exception):
+    """An error by a user of a cmdln-based tool/shell."""
+    pass
+
+
+
+#---- public methods and classes
+
+def alias(*aliases):
+    """Decorator to add aliases for Cmdln.do_* command handlers.
+    
+    Example:
+        class MyShell(cmdln.Cmdln):
+            @cmdln.alias("!", "sh")
+            def do_shell(self, argv):
+                #...implement 'shell' command
+    """
+    def decorate(f):
+        if not hasattr(f, "aliases"):
+            f.aliases = []
+        f.aliases += aliases
+        return f
+    return decorate
+
+
+class RawCmdln(cmd.Cmd):
+    """An improved (on cmd.Cmd) framework for building multi-subcommand
+    scripts (think "svn" & "cvs") and simple shells (think "pdb" and
+    "gdb").
+
+    A simple example:
+
+        import cmdln
+
+        class MySVN(cmdln.RawCmdln):
+            name = "svn"
+
+            @cmdln.aliases('stat', 'st')
+            def do_status(self, argv):
+                print "handle 'svn status' command"
+
+        if __name__ == "__main__":
+            shell = MySVN()
+            retval = shell.main()
+            sys.exit(retval)
+
+    See <http://trentm.com/projects/cmdln> for more information.
+    """
+    name = None      # if unset, defaults basename(sys.argv[0])
+    prompt = None    # if unset, defaults to self.name+"> "
+    version = None   # if set, default top-level options include --version
+
+    # Default messages for some 'help' command error cases.
+    # They are interpolated with one arg: the command.
+    nohelp = "no help on '%s'"
+    unknowncmd = "unknown command: '%s'"
+
+    helpindent = '' # string with which to indent help output
+
+    def __init__(self, completekey='tab', 
+                 stdin=None, stdout=None, stderr=None):
+        """Cmdln(completekey='tab', stdin=None, stdout=None, stderr=None)
+
+        The optional argument 'completekey' is the readline name of a
+        completion key; it defaults to the Tab key. If completekey is
+        not None and the readline module is available, command completion
+        is done automatically.
+        
+        The optional arguments 'stdin', 'stdout' and 'stderr' specify
+        alternate input, output and error output file objects; if not
+        specified, sys.* are used.
+        
+        If 'stdout' but not 'stderr' is specified, stdout is used for
+        error output. This is to provide least surprise for users used
+        to only the 'stdin' and 'stdout' options with cmd.Cmd.
+        """
+        import sys
+        if self.name is None:
+            self.name = os.path.basename(sys.argv[0])
+        if self.prompt is None:
+            self.prompt = self.name+"> "
+        self._name_str = self._str(self.name)
+        self._prompt_str = self._str(self.prompt)
+        if stdin is not None:
+            self.stdin = stdin
+        else:
+            self.stdin = sys.stdin
+        if stdout is not None:
+            self.stdout = stdout
+        else:
+            self.stdout = sys.stdout
+        if stderr is not None:
+            self.stderr = stderr
+        elif stdout is not None:
+            self.stderr = stdout
+        else:
+            self.stderr = sys.stderr
+        self.cmdqueue = []
+        self.completekey = completekey
+        self.cmdlooping = False
+
+    def get_optparser(self):
+        """Hook for subclasses to set the option parser for the
+        top-level command/shell.
+
+        This option parser is used retrieved and used by `.main()' to
+        handle top-level options.
+
+        The default implements a single '-h|--help' option. Sub-classes
+        can return None to have no options at the top-level. Typically
+        an instance of CmdlnOptionParser should be returned.
+        """
+        version = (self.version is not None 
+                    and "%s %s" % (self._name_str, self.version)
+                    or None)
+        return CmdlnOptionParser(self, version=version)
+
+    def postoptparse(self):
+        """Hook method executed just after `.main()' parses top-level
+        options.
+
+        When called `self.options' holds the results of the option parse.
+        """
+        pass
+
+    def main(self, argv=None, loop=LOOP_NEVER):
+        """A possible mainline handler for a script, like so:
+
+            import cmdln
+            class MyCmd(cmdln.Cmdln):
+                name = "mycmd"
+                ...
+            
+            if __name__ == "__main__":
+                MyCmd().main()
+
+        By default this will use sys.argv to issue a single command to
+        'MyCmd', then exit. The 'loop' argument can be use to control
+        interactive shell behaviour.
+        
+        Arguments:
+            "argv" (optional, default sys.argv) is the command to run.
+                It must be a sequence, where the first element is the
+                command name and subsequent elements the args for that
+                command.
+            "loop" (optional, default LOOP_NEVER) is a constant
+                indicating if a command loop should be started (i.e. an
+                interactive shell). Valid values (constants on this module):
+                    LOOP_ALWAYS     start loop and run "argv", if any
+                    LOOP_NEVER      run "argv" (or .emptyline()) and exit
+                    LOOP_IF_EMPTY   run "argv", if given, and exit;
+                                    otherwise, start loop
+        """
+        if argv is None:
+            import sys
+            argv = sys.argv
+        else:
+            argv = argv[:] # don't modify caller's list
+
+        self.optparser = self.get_optparser()
+        if self.optparser: # i.e. optparser=None means don't process for opts
+            try:
+                self.options, args = self.optparser.parse_args(argv[1:])
+            except CmdlnUserError, ex:
+                msg = "%s: %s\nTry '%s help' for info.\n"\
+                      % (self.name, ex, self.name)
+                self.stderr.write(self._str(msg))
+                self.stderr.flush()
+                return 1
+            except StopOptionProcessing, ex:
+                return 0
+        else:
+            self.options, args = None, argv[1:]
+        self.postoptparse()
+
+        if loop == LOOP_ALWAYS:
+            if args:
+                self.cmdqueue.append(args)
+            return self.cmdloop()
+        elif loop == LOOP_NEVER:
+            if args:
+                return self.cmd(args)
+            else:
+                return self.emptyline()
+        elif loop == LOOP_IF_EMPTY:
+            if args:
+                return self.cmd(args)
+            else:
+                return self.cmdloop()
+
+    def cmd(self, argv):
+        """Run one command and exit.
+        
+            "argv" is the arglist for the command to run. argv[0] is the
+                command to run. If argv is an empty list then the
+                'emptyline' handler is run.
+
+        Returns the return value from the command handler.
+        """
+        assert isinstance(argv, (list, tuple)), \
+                "'argv' is not a sequence: %r" % argv
+        retval = None
+        try:
+            argv = self.precmd(argv)
+            retval = self.onecmd(argv)
+            self.postcmd(argv)
+        except:
+            if not self.cmdexc(argv):
+                raise
+            retval = 1
+        return retval
+
+    def _str(self, s):
+        """Safely convert the given str/unicode to a string for printing."""
+        try:
+            return str(s)
+        except UnicodeError:
+            #XXX What is the proper encoding to use here? 'utf-8' seems
+            #    to work better than "getdefaultencoding" (usually
+            #    'ascii'), on OS X at least.
+            #import sys
+            #return s.encode(sys.getdefaultencoding(), "replace")
+            return s.encode("utf-8", "replace")
+
+    def cmdloop(self, intro=None):
+        """Repeatedly issue a prompt, accept input, parse into an argv, and
+        dispatch (via .precmd(), .onecmd() and .postcmd()), passing them
+        the argv. In other words, start a shell.
+        
+            "intro" (optional) is a introductory message to print when
+                starting the command loop. This overrides the class
+                "intro" attribute, if any.
+        """
+        self.cmdlooping = True
+        self.preloop()
+        if self.use_rawinput and self.completekey:
+            try:
+                import readline
+                self.old_completer = readline.get_completer()
+                readline.set_completer(self.complete)
+                readline.parse_and_bind(self.completekey+": complete")
+            except ImportError:
+                pass
+        try:
+            if intro is None:
+                intro = self.intro
+            if intro:
+                intro_str = self._str(intro)
+                self.stdout.write(intro_str+'\n')
+            self.stop = False
+            retval = None
+            while not self.stop:
+                if self.cmdqueue:
+                    argv = self.cmdqueue.pop(0)
+                    assert isinstance(argv, (list, tuple)), \
+                            "item on 'cmdqueue' is not a sequence: %r" % argv
+                else:
+                    if self.use_rawinput:
+                        try:
+                            line = raw_input(self._prompt_str)
+                        except EOFError:
+                            line = 'EOF'
+                    else:
+                        self.stdout.write(self._prompt_str)
+                        self.stdout.flush()
+                        line = self.stdin.readline()
+                        if not len(line):
+                            line = 'EOF'
+                        else:
+                            line = line[:-1] # chop '\n'
+                    argv = line2argv(line)
+                try:
+                    argv = self.precmd(argv)
+                    retval = self.onecmd(argv)
+                    self.postcmd(argv)
+                except:
+                    if not self.cmdexc(argv):
+                        raise
+                    retval = 1
+                self.lastretval = retval
+            self.postloop()
+        finally:
+            if self.use_rawinput and self.completekey:
+                try:
+                    import readline
+                    readline.set_completer(self.old_completer)
+                except ImportError:
+                    pass
+        self.cmdlooping = False
+        return retval
+
+    def precmd(self, argv):
+        """Hook method executed just before the command argv is
+        interpreted, but after the input prompt is generated and issued.
+
+            "argv" is the cmd to run.
+            
+        Returns an argv to run (i.e. this method can modify the command
+        to run).
+        """
+        return argv
+
+    def postcmd(self, argv):
+        """Hook method executed just after a command dispatch is finished.
+        
+            "argv" is the command that was run.
+        """
+        pass
+
+    def cmdexc(self, argv):
+        """Called if an exception is raised in any of precmd(), onecmd(),
+        or postcmd(). If True is returned, the exception is deemed to have
+        been dealt with. Otherwise, the exception is re-raised.
+
+        The default implementation handles CmdlnUserError's, which
+        typically correspond to user error in calling commands (as
+        opposed to programmer error in the design of the script using
+        cmdln.py).
+        """
+        import sys
+        type, exc, traceback = sys.exc_info()
+        if isinstance(exc, CmdlnUserError):
+            msg = "%s %s: %s\nTry '%s help %s' for info.\n"\
+                  % (self.name, argv[0], exc, self.name, argv[0])
+            self.stderr.write(self._str(msg))
+            self.stderr.flush()
+            return True
+
+    def onecmd(self, argv):
+        if not argv:
+            return self.emptyline()
+        self.lastcmd = argv
+        cmdname = self._get_canonical_cmd_name(argv[0])
+        if cmdname:
+            handler = self._get_cmd_handler(cmdname)
+            if handler:
+                return self._dispatch_cmd(handler, argv)
+        return self.default(argv)
+
+    def _dispatch_cmd(self, handler, argv):
+        return handler(argv)
+
+    def default(self, argv):
+        """Hook called to handle a command for which there is no handler.
+
+            "argv" is the command and arguments to run.
+        
+        The default implementation writes and error message to stderr
+        and returns an error exit status.
+
+        Returns a numeric command exit status.
+        """
+        errmsg = self._str(self.unknowncmd % (argv[0],))
+        if self.cmdlooping:
+            self.stderr.write(errmsg+"\n")
+        else:
+            self.stderr.write("%s: %s\nTry '%s help' for info.\n"
+                              % (self._name_str, errmsg, self._name_str))
+        self.stderr.flush()
+        return 1
+
+    def parseline(self, line):
+        # This is used by Cmd.complete (readline completer function) to
+        # massage the current line buffer before completion processing.
+        # We override to drop special '!' handling.
+        line = line.strip()
+        if not line:
+            return None, None, line
+        elif line[0] == '?':
+            line = 'help ' + line[1:]
+        i, n = 0, len(line)
+        while i < n and line[i] in self.identchars: i = i+1
+        cmd, arg = line[:i], line[i:].strip()
+        return cmd, arg, line
+
+    def helpdefault(self, cmd, known):
+        """Hook called to handle help on a command for which there is no
+        help handler.
+
+            "cmd" is the command name on which help was requested.
+            "known" is a boolean indicating if this command is known
+                (i.e. if there is a handler for it).
+        
+        Returns a return code.
+        """
+        if known:
+            msg = self._str(self.nohelp % (cmd,))
+            if self.cmdlooping:
+                self.stderr.write(msg + '\n')
+            else:
+                self.stderr.write("%s: %s\n" % (self.name, msg))
+        else:
+            msg = self.unknowncmd % (cmd,)
+            if self.cmdlooping:
+                self.stderr.write(msg + '\n')
+            else:
+                self.stderr.write("%s: %s\n"
+                                  "Try '%s help' for info.\n"
+                                  % (self.name, msg, self.name))
+        self.stderr.flush()
+        return 1
+
+    def do_help(self, argv):
+        """${cmd_name}: give detailed help on a specific sub-command
+
+        Usage:
+            ${name} help [COMMAND]
+        """
+        if len(argv) > 1: # asking for help on a particular command
+            doc = None
+            cmdname = self._get_canonical_cmd_name(argv[1]) or argv[1]
+            if not cmdname:
+                return self.helpdefault(argv[1], False)
+            else:
+                helpfunc = getattr(self, "help_"+cmdname, None)
+                if helpfunc:
+                    doc = helpfunc()
+                else:
+                    handler = self._get_cmd_handler(cmdname)
+                    if handler:
+                        doc = handler.__doc__
+                    if doc is None:
+                        return self.helpdefault(argv[1], handler != None)
+        else: # bare "help" command
+            doc = self.__class__.__doc__  # try class docstring
+            if doc is None:
+                # Try to provide some reasonable useful default help.
+                if self.cmdlooping: prefix = ""
+                else:               prefix = self.name+' '
+                doc = """Usage:
+                    %sCOMMAND [ARGS...]
+                    %shelp [COMMAND]
+
+                ${option_list}
+                ${command_list}
+                ${help_list}
+                """ % (prefix, prefix)
+            cmdname = None
+
+        if doc: # *do* have help content, massage and print that
+            doc = self._help_reindent(doc)
+            doc = self._help_preprocess(doc, cmdname)
+            doc = doc.rstrip() + '\n' # trim down trailing space
+            self.stdout.write(self._str(doc))
+            self.stdout.flush()
+    do_help.aliases = ["?"]
+
+    def _help_reindent(self, help, indent=None):
+        """Hook to re-indent help strings before writing to stdout.
+
+            "help" is the help content to re-indent
+            "indent" is a string with which to indent each line of the
+                help content after normalizing. If unspecified or None
+                then the default is use: the 'self.helpindent' class
+                attribute. By default this is the empty string, i.e.
+                no indentation.
+
+        By default, all common leading whitespace is removed and then
+        the lot is indented by 'self.helpindent'. When calculating the
+        common leading whitespace the first line is ignored -- hence
+        help content for Conan can be written as follows and have the
+        expected indentation:
+
+            def do_crush(self, ...):
+                '''${cmd_name}: crush your enemies, see them driven before you...
+
+                c.f. Conan the Barbarian'''
+        """
+        if indent is None:
+            indent = self.helpindent
+        lines = help.splitlines(0)
+        _dedentlines(lines, skip_first_line=True)
+        lines = [(indent+line).rstrip() for line in lines]
+        return '\n'.join(lines)
+
+    def _help_preprocess(self, help, cmdname):
+        """Hook to preprocess a help string before writing to stdout.
+
+            "help" is the help string to process.
+            "cmdname" is the canonical sub-command name for which help
+                is being given, or None if the help is not specific to a
+                command.
+
+        By default the following template variables are interpolated in
+        help content. (Note: these are similar to Python 2.4's
+        string.Template interpolation but not quite.)
+
+        ${name}
+            The tool's/shell's name, i.e. 'self.name'.
+        ${option_list}
+            A formatted table of options for this shell/tool.
+        ${command_list}
+            A formatted table of available sub-commands.
+        ${help_list}
+            A formatted table of additional help topics (i.e. 'help_*'
+            methods with no matching 'do_*' method).
+        ${cmd_name}
+            The name (and aliases) for this sub-command formatted as:
+            "NAME (ALIAS1, ALIAS2, ...)".
+        ${cmd_usage}
+            A formatted usage block inferred from the command function
+            signature.
+        ${cmd_option_list}
+            A formatted table of options for this sub-command. (This is
+            only available for commands using the optparse integration,
+            i.e.  using @cmdln.option decorators or manually setting the
+            'optparser' attribute on the 'do_*' method.)
+
+        Returns the processed help. 
+        """
+        preprocessors = {
+            "${name}":            self._help_preprocess_name,
+            "${option_list}":     self._help_preprocess_option_list,
+            "${command_list}":    self._help_preprocess_command_list,
+            "${help_list}":       self._help_preprocess_help_list,
+            "${cmd_name}":        self._help_preprocess_cmd_name,
+            "${cmd_usage}":       self._help_preprocess_cmd_usage,
+            "${cmd_option_list}": self._help_preprocess_cmd_option_list,
+        }
+
+        for marker, preprocessor in preprocessors.items():
+            if marker in help:
+                help = preprocessor(help, cmdname)
+        return help
+
+    def _help_preprocess_name(self, help, cmdname=None):
+        return help.replace("${name}", self.name)
+
+    def _help_preprocess_option_list(self, help, cmdname=None):
+        marker = "${option_list}"
+        indent, indent_width = _get_indent(marker, help)
+        suffix = _get_trailing_whitespace(marker, help)
+
+        if self.optparser:
+            # Setup formatting options and format.
+            # - Indentation of 4 is better than optparse default of 2.
+            #   C.f. Damian Conway's discussion of this in Perl Best
+            #   Practices.
+            self.optparser.formatter.indent_increment = 4
+            self.optparser.formatter.current_indent = indent_width
+            block = self.optparser.format_option_help() + '\n'
+        else:
+            block = ""
+            
+        help = help.replace(indent+marker+suffix, block, 1)
+        return help
+
+
+    def _help_preprocess_command_list(self, help, cmdname=None):
+        marker = "${command_list}"
+        indent, indent_width = _get_indent(marker, help)
+        suffix = _get_trailing_whitespace(marker, help)
+
+        # Find any aliases for commands.
+        token2canonical = self._get_canonical_map()
+        aliases = {}
+        for token, cmdname in token2canonical.items():
+            if token == cmdname: continue
+            aliases.setdefault(cmdname, []).append(token)
+
+        # Get the list of (non-hidden) commands and their
+        # documentation, if any.
+        cmdnames = {} # use a dict to strip duplicates
+        for attr in self.get_names():
+            if attr.startswith("do_"):
+                cmdnames[attr[3:]] = True
+        cmdnames = cmdnames.keys()
+        cmdnames.sort()
+        linedata = []
+        for cmdname in cmdnames:
+            if aliases.get(cmdname):
+                a = aliases[cmdname]
+                a.sort()
+                cmdstr = "%s (%s)" % (cmdname, ", ".join(a))
+            else:
+                cmdstr = cmdname
+            doc = None
+            try:
+                helpfunc = getattr(self, 'help_'+cmdname)
+            except AttributeError:
+                handler = self._get_cmd_handler(cmdname)
+                if handler:
+                    doc = handler.__doc__
+            else:
+                doc = helpfunc()
+                
+            # Strip "${cmd_name}: " from the start of a command's doc. Best
+            # practice dictates that command help strings begin with this, but
+            # it isn't at all wanted for the command list.
+            to_strip = "${cmd_name}:"
+            if doc and doc.startswith(to_strip):
+                #log.debug("stripping %r from start of %s's help string",
+                #          to_strip, cmdname)
+                doc = doc[len(to_strip):].lstrip()
+            linedata.append( (cmdstr, doc) )
+
+        if linedata:
+            subindent = indent + ' '*4
+            lines = _format_linedata(linedata, subindent, indent_width+4)
+            block = indent + "Commands:\n" \
+                    + '\n'.join(lines) + "\n\n"
+            help = help.replace(indent+marker+suffix, block, 1)
+        return help
+
+    def _gen_names_and_attrs(self):
+        # Inheritance says we have to look in class and
+        # base classes; order is not important.
+        names = []
+        classes = [self.__class__]
+        while classes:
+            aclass = classes.pop(0)
+            if aclass.__bases__:
+                classes = classes + list(aclass.__bases__)
+            for name in dir(aclass):
+                yield (name, getattr(aclass, name))
+
+    def _help_preprocess_help_list(self, help, cmdname=None):
+        marker = "${help_list}"
+        indent, indent_width = _get_indent(marker, help)
+        suffix = _get_trailing_whitespace(marker, help)
+
+        # Determine the additional help topics, if any.
+        helpnames = {}
+        token2cmdname = self._get_canonical_map()
+        for attrname, attr in self._gen_names_and_attrs():
+            if not attrname.startswith("help_"): continue
+            helpname = attrname[5:]
+            if helpname not in token2cmdname:
+                helpnames[helpname] = attr
+
+        if helpnames:
+            linedata = [(n, a.__doc__ or "") for n, a in helpnames.items()]
+            linedata.sort()
+
+            subindent = indent + ' '*4
+            lines = _format_linedata(linedata, subindent, indent_width+4)
+            block = (indent
+                    + "Additional help topics (run `%s help TOPIC'):\n" % self.name
+                    + '\n'.join(lines)
+                    + "\n\n")
+        else:
+            block = ''
+        help = help.replace(indent+marker+suffix, block, 1)
+        return help
+
+    def _help_preprocess_cmd_name(self, help, cmdname=None):
+        marker = "${cmd_name}"
+        handler = self._get_cmd_handler(cmdname)
+        if not handler:
+            raise CmdlnError("cannot preprocess '%s' into help string: "
+                             "could not find command handler for %r" 
+                             % (marker, cmdname))
+        s = cmdname
+        if hasattr(handler, "aliases"):
+            s += " (%s)" % (", ".join(handler.aliases))
+        help = help.replace(marker, s)
+        return help
+
+    #TODO: this only makes sense as part of the Cmdln class.
+    #      Add hooks to add help preprocessing template vars and put
+    #      this one on that class.
+    def _help_preprocess_cmd_usage(self, help, cmdname=None):
+        marker = "${cmd_usage}"
+        handler = self._get_cmd_handler(cmdname)
+        if not handler:
+            raise CmdlnError("cannot preprocess '%s' into help string: "
+                             "could not find command handler for %r" 
+                             % (marker, cmdname))
+        indent, indent_width = _get_indent(marker, help)
+        suffix = _get_trailing_whitespace(marker, help)
+
+        # Extract the introspection bits we need.
+        func = handler.im_func
+        if func.func_defaults:
+            func_defaults = list(func.func_defaults)
+        else:
+            func_defaults = []
+        co_argcount = func.func_code.co_argcount
+        co_varnames = func.func_code.co_varnames
+        co_flags = func.func_code.co_flags
+        CO_FLAGS_ARGS = 4
+        CO_FLAGS_KWARGS = 8
+
+        # Adjust argcount for possible *args and **kwargs arguments.
+        argcount = co_argcount
+        if co_flags & CO_FLAGS_ARGS:   argcount += 1
+        if co_flags & CO_FLAGS_KWARGS: argcount += 1
+
+        # Determine the usage string.
+        usage = "%s %s" % (self.name, cmdname)
+        if argcount <= 2:   # handler ::= do_FOO(self, argv)
+            usage += " [ARGS...]"
+        elif argcount >= 3: # handler ::= do_FOO(self, subcmd, opts, ...)
+            argnames = list(co_varnames[3:argcount])
+            tail = ""
+            if co_flags & CO_FLAGS_KWARGS:
+                name = argnames.pop(-1)
+                import warnings
+                # There is no generally accepted mechanism for passing
+                # keyword arguments from the command line. Could
+                # *perhaps* consider: arg=value arg2=value2 ...
+                warnings.warn("argument '**%s' on '%s.%s' command "
+                              "handler will never get values" 
+                              % (name, self.__class__.__name__,
+                                 func.func_name))
+            if co_flags & CO_FLAGS_ARGS:
+                name = argnames.pop(-1)
+                tail = "[%s...]" % name.upper()
+            while func_defaults:
+                func_defaults.pop(-1)
+                name = argnames.pop(-1)
+                tail = "[%s%s%s]" % (name.upper(), (tail and ' ' or ''), tail)
+            while argnames:
+                name = argnames.pop(-1)
+                tail = "%s %s" % (name.upper(), tail)
+            usage += ' ' + tail
+
+        block_lines = [
+            self.helpindent + "Usage:",
+            self.helpindent + ' '*4 + usage
+        ]
+        block = '\n'.join(block_lines) + '\n\n'
+
+        help = help.replace(indent+marker+suffix, block, 1)
+        return help
+
+    #TODO: this only makes sense as part of the Cmdln class.
+    #      Add hooks to add help preprocessing template vars and put
+    #      this one on that class.
+    def _help_preprocess_cmd_option_list(self, help, cmdname=None):
+        marker = "${cmd_option_list}"
+        handler = self._get_cmd_handler(cmdname)
+        if not handler:
+            raise CmdlnError("cannot preprocess '%s' into help string: "
+                             "could not find command handler for %r" 
+                             % (marker, cmdname))
+        indent, indent_width = _get_indent(marker, help)
+        suffix = _get_trailing_whitespace(marker, help)
+        if hasattr(handler, "optparser"):
+            # Setup formatting options and format.
+            # - Indentation of 4 is better than optparse default of 2.
+            #   C.f. Damian Conway's discussion of this in Perl Best
+            #   Practices.
+            handler.optparser.formatter.indent_increment = 4
+            handler.optparser.formatter.current_indent = indent_width
+            block = handler.optparser.format_option_help() + '\n'
+        else:
+            block = ""
+
+        help = help.replace(indent+marker+suffix, block, 1)
+        return help
+
+    def _get_canonical_cmd_name(self, token):
+        map = self._get_canonical_map()
+        return map.get(token, None)
+
+    def _get_canonical_map(self):
+        """Return a mapping of available command names and aliases to
+        their canonical command name.
+        """
+        cacheattr = "_token2canonical"
+        if not hasattr(self, cacheattr):
+            # Get the list of commands and their aliases, if any.
+            token2canonical = {}
+            cmd2funcname = {} # use a dict to strip duplicates
+            for attr in self.get_names():
+                if attr.startswith("do_"):    cmdname = attr[3:]
+                elif attr.startswith("_do_"): cmdname = attr[4:]
+                else:
+                    continue
+                cmd2funcname[cmdname] = attr
+                token2canonical[cmdname] = cmdname
+            for cmdname, funcname in cmd2funcname.items(): # add aliases
+                func = getattr(self, funcname)
+                aliases = getattr(func, "aliases", [])
+                for alias in aliases:
+                    if alias in cmd2funcname:
+                        import warnings
+                        warnings.warn("'%s' alias for '%s' command conflicts "
+                                      "with '%s' handler"
+                                      % (alias, cmdname, cmd2funcname[alias]))
+                        continue
+                    token2canonical[alias] = cmdname
+            setattr(self, cacheattr, token2canonical)
+        return getattr(self, cacheattr)
+
+    def _get_cmd_handler(self, cmdname):
+        handler = None
+        try:
+            handler = getattr(self, 'do_' + cmdname)
+        except AttributeError:
+            try:
+                # Private command handlers begin with "_do_".
+                handler = getattr(self, '_do_' + cmdname)
+            except AttributeError:
+                pass
+        return handler
+
+    def _do_EOF(self, argv):
+        # Default EOF handler
+        # Note: an actual EOF is redirected to this command.
+        #TODO: separate name for this. Currently it is available from
+        #      command-line. Is that okay?
+        self.stdout.write('\n')
+        self.stdout.flush()
+        self.stop = True
+
+    def emptyline(self):
+        # Different from cmd.Cmd: don't repeat the last command for an
+        # emptyline.
+        if self.cmdlooping:
+            pass
+        else:
+            return self.do_help(["help"])
+
+
+#---- optparse.py extension to fix (IMO) some deficiencies
+#
+# See the class _OptionParserEx docstring for details.
+#
+
+class StopOptionProcessing(Exception):
+    """Indicate that option *and argument* processing should stop
+    cleanly. This is not an error condition. It is similar in spirit to
+    StopIteration. This is raised by _OptionParserEx's default "help"
+    and "version" option actions and can be raised by custom option
+    callbacks too.
+    
+    Hence the typical CmdlnOptionParser (a subclass of _OptionParserEx)
+    usage is:
+
+        parser = CmdlnOptionParser(mycmd)
+        parser.add_option("-f", "--force", dest="force")
+        ...
+        try:
+            opts, args = parser.parse_args()
+        except StopOptionProcessing:
+            # normal termination, "--help" was probably given
+            sys.exit(0)
+    """
+
+class _OptionParserEx(optparse.OptionParser):
+    """An optparse.OptionParser that uses exceptions instead of sys.exit.
+
+    This class is an extension of optparse.OptionParser that differs
+    as follows:
+    - Correct (IMO) the default OptionParser error handling to never
+      sys.exit(). Instead OptParseError exceptions are passed through.
+    - Add the StopOptionProcessing exception (a la StopIteration) to
+      indicate normal termination of option processing.
+      See StopOptionProcessing's docstring for details.
+
+    I'd also like to see the following in the core optparse.py, perhaps
+    as a RawOptionParser which would serve as a base class for the more
+    generally used OptionParser (that works as current):
+    - Remove the implicit addition of the -h|--help and --version
+      options. They can get in the way (e.g. if want '-?' and '-V' for
+      these as well) and it is not hard to do:
+        optparser.add_option("-h", "--help", action="help")
+        optparser.add_option("--version", action="version")
+      These are good practices, just not valid defaults if they can
+      get in the way.
+    """
+    def error(self, msg):
+        raise optparse.OptParseError(msg)
+
+    def exit(self, status=0, msg=None):
+        if status == 0:
+            raise StopOptionProcessing(msg)
+        else:
+            #TODO: don't lose status info here
+            raise optparse.OptParseError(msg)
+
+
+
+#---- optparse.py-based option processing support
+
+class CmdlnOptionParser(_OptionParserEx):
+    """An optparse.OptionParser class more appropriate for top-level
+    Cmdln options. For parsing of sub-command options, see
+    SubCmdOptionParser.
+
+    Changes:
+    - disable_interspersed_args() by default, because a Cmdln instance
+      has sub-commands which may themselves have options.
+    - Redirect print_help() to the Cmdln.do_help() which is better
+      equiped to handle the "help" action.
+    - error() will raise a CmdlnUserError: OptionParse.error() is meant
+      to be called for user errors. Raising a well-known error here can
+      make error handling clearer.
+    - Also see the changes in _OptionParserEx.
+    """
+    def __init__(self, cmdln, **kwargs):
+        self.cmdln = cmdln
+        kwargs["prog"] = self.cmdln.name
+        _OptionParserEx.__init__(self, **kwargs)
+        self.disable_interspersed_args()
+
+    def print_help(self, file=None):
+        self.cmdln.onecmd(["help"])
+
+    def error(self, msg):
+        raise CmdlnUserError(msg)
+
+
+class SubCmdOptionParser(_OptionParserEx):
+    def set_cmdln_info(self, cmdln, subcmd):
+        """Called by Cmdln to pass relevant info about itself needed
+        for print_help().
+        """
+        self.cmdln = cmdln
+        self.subcmd = subcmd
+
+    def print_help(self, file=None):
+        self.cmdln.onecmd(["help", self.subcmd])
+
+    def error(self, msg):
+        raise CmdlnUserError(msg)
+
+
+def option(*args, **kwargs):
+    """Decorator to add an option to the optparser argument of a Cmdln
+    subcommand.
+    
+    Example:
+        class MyShell(cmdln.Cmdln):
+            @cmdln.option("-f", "--force", help="force removal")
+            def do_remove(self, subcmd, opts, *args):
+                #...
+    """
+    #XXX Is there a possible optimization for many options to not have a
+    #    large stack depth here?
+    def decorate(f):
+        if not hasattr(f, "optparser"):
+            f.optparser = SubCmdOptionParser()
+        f.optparser.add_option(*args, **kwargs)
+        return f
+    return decorate
+
+
+class Cmdln(RawCmdln):
+    """An improved (on cmd.Cmd) framework for building multi-subcommand
+    scripts (think "svn" & "cvs") and simple shells (think "pdb" and
+    "gdb").
+
+    A simple example:
+
+        import cmdln
+
+        class MySVN(cmdln.Cmdln):
+            name = "svn"
+
+            @cmdln.aliases('stat', 'st')
+            @cmdln.option('-v', '--verbose', action='store_true'
+                          help='print verbose information')
+            def do_status(self, subcmd, opts, *paths):
+                print "handle 'svn status' command"
+
+            #...
+
+        if __name__ == "__main__":
+            shell = MySVN()
+            retval = shell.main()
+            sys.exit(retval)
+
+    'Cmdln' extends 'RawCmdln' by providing optparse option processing
+    integration.  See this class' _dispatch_cmd() docstring and
+    <http://trentm.com/projects/cmdln> for more information.
+    """
+    def _dispatch_cmd(self, handler, argv):
+        """Introspect sub-command handler signature to determine how to
+        dispatch the command. The raw handler provided by the base
+        'RawCmdln' class is still supported:
+
+            def do_foo(self, argv):
+                # 'argv' is the vector of command line args, argv[0] is
+                # the command name itself (i.e. "foo" or an alias)
+                pass
+
+        In addition, if the handler has more than 2 arguments option
+        processing is automatically done (using optparse):
+
+            @cmdln.option('-v', '--verbose', action='store_true')
+            def do_bar(self, subcmd, opts, *args):
+                # subcmd = <"bar" or an alias>
+                # opts = <an optparse.Values instance>
+                if opts.verbose:
+                    print "lots of debugging output..."
+                # args = <tuple of arguments>
+                for arg in args:
+                    bar(arg)
+
+        TODO: explain that "*args" can be other signatures as well.
+
+        The `cmdln.option` decorator corresponds to an `add_option()`
+        method call on an `optparse.OptionParser` instance.
+
+        You can declare a specific number of arguments:
+
+            @cmdln.option('-v', '--verbose', action='store_true')
+            def do_bar2(self, subcmd, opts, bar_one, bar_two):
+                #...
+
+        and an appropriate error message will be raised/printed if the
+        command is called with a different number of args.
+        """
+        co_argcount = handler.im_func.func_code.co_argcount
+        if co_argcount == 2:   # handler ::= do_foo(self, argv)
+            return handler(argv)
+        elif co_argcount >= 3: # handler ::= do_foo(self, subcmd, opts, ...)
+            try:
+                optparser = handler.optparser
+            except AttributeError:
+                optparser = handler.im_func.optparser = SubCmdOptionParser()
+            assert isinstance(optparser, SubCmdOptionParser)
+            optparser.set_cmdln_info(self, argv[0])
+            try:
+                opts, args = optparser.parse_args(argv[1:])
+            except StopOptionProcessing:
+                #TODO: this doesn't really fly for a replacement of
+                #      optparse.py behaviour, does it?
+                return 0 # Normal command termination
+
+            try:
+                return handler(argv[0], opts, *args)
+            except TypeError, ex:
+                # Some TypeError's are user errors:
+                #   do_foo() takes at least 4 arguments (3 given)
+                #   do_foo() takes at most 5 arguments (6 given)
+                #   do_foo() takes exactly 5 arguments (6 given)
+                # Raise CmdlnUserError for these with a suitably
+                # massaged error message.
+                import sys
+                tb = sys.exc_info()[2] # the traceback object
+                if tb.tb_next is not None:
+                    # If the traceback is more than one level deep, then the
+                    # TypeError do *not* happen on the "handler(...)" call
+                    # above. In that we don't want to handle it specially
+                    # here: it would falsely mask deeper code errors.
+                    raise
+                msg = ex.args[0]
+                match = _INCORRECT_NUM_ARGS_RE.search(msg)
+                if match:
+                    msg = list(match.groups())
+                    msg[1] = int(msg[1]) - 3
+                    if msg[1] == 1:
+                        msg[2] = msg[2].replace("arguments", "argument")
+                    msg[3] = int(msg[3]) - 3
+                    msg = ''.join(map(str, msg))
+                    raise CmdlnUserError(msg)
+                else:
+                    raise
+        else:
+            raise CmdlnError("incorrect argcount for %s(): takes %d, must "
+                             "take 2 for 'argv' signature or 3+ for 'opts' "
+                             "signature" % (handler.__name__, co_argcount))
+        
+
+
+#---- internal support functions
+
+def _format_linedata(linedata, indent, indent_width):
+    """Format specific linedata into a pleasant layout.
+    
+        "linedata" is a list of 2-tuples of the form:
+            (<item-display-string>, <item-docstring>)
+        "indent" is a string to use for one level of indentation
+        "indent_width" is a number of columns by which the
+            formatted data will be indented when printed.
+
+    The <item-display-string> column is held to 15 columns.
+    """
+    lines = []
+    WIDTH = 78 - indent_width
+    SPACING = 2
+    NAME_WIDTH_LOWER_BOUND = 13
+    NAME_WIDTH_UPPER_BOUND = 16
+    NAME_WIDTH = max([len(s) for s,d in linedata])
+    if NAME_WIDTH < NAME_WIDTH_LOWER_BOUND:
+        NAME_WIDTH = NAME_WIDTH_LOWER_BOUND
+    else:
+        NAME_WIDTH = NAME_WIDTH_UPPER_BOUND
+
+    DOC_WIDTH = WIDTH - NAME_WIDTH - SPACING
+    for namestr, doc in linedata:
+        line = indent + namestr
+        if len(namestr) <= NAME_WIDTH:
+            line += ' ' * (NAME_WIDTH + SPACING - len(namestr))
+        else:
+            lines.append(line)
+            line = indent + ' ' * (NAME_WIDTH + SPACING)
+        line += _summarize_doc(doc, DOC_WIDTH)
+        lines.append(line.rstrip())
+    return lines
+
+def _summarize_doc(doc, length=60):
+    r"""Parse out a short one line summary from the given doclines.
+    
+        "doc" is the doc string to summarize.
+        "length" is the max length for the summary
+
+    >>> _summarize_doc("this function does this")
+    'this function does this'
+    >>> _summarize_doc("this function does this", 10)
+    'this fu...'
+    >>> _summarize_doc("this function does this\nand that")
+    'this function does this and that'
+    >>> _summarize_doc("this function does this\n\nand that")
+    'this function does this'
+    """
+    import re
+    if doc is None:
+        return ""
+    assert length > 3, "length <= 3 is absurdly short for a doc summary"
+    doclines = doc.strip().splitlines(0)
+    if not doclines:
+        return ""
+
+    summlines = []
+    for i, line in enumerate(doclines):
+        stripped = line.strip()
+        if not stripped:
+            break
+        summlines.append(stripped)
+        if len(''.join(summlines)) >= length:
+            break
+
+    summary = ' '.join(summlines)
+    if len(summary) > length:
+        summary = summary[:length-3] + "..." 
+    return summary
+
+
+def line2argv(line):
+    r"""Parse the given line into an argument vector.
+    
+        "line" is the line of input to parse.
+
+    This may get niggly when dealing with quoting and escaping. The
+    current state of this parsing may not be completely thorough/correct
+    in this respect.
+    
+    >>> from cmdln import line2argv
+    >>> line2argv("foo")
+    ['foo']
+    >>> line2argv("foo bar")
+    ['foo', 'bar']
+    >>> line2argv("foo bar ")
+    ['foo', 'bar']
+    >>> line2argv(" foo bar")
+    ['foo', 'bar']
+
+    Quote handling:
+    
+    >>> line2argv("'foo bar'")
+    ['foo bar']
+    >>> line2argv('"foo bar"')
+    ['foo bar']
+    >>> line2argv(r'"foo\"bar"')
+    ['foo"bar']
+    >>> line2argv("'foo bar' spam")
+    ['foo bar', 'spam']
+    >>> line2argv("'foo 'bar spam")
+    ['foo bar', 'spam']
+    
+    >>> line2argv('some\tsimple\ttests')
+    ['some', 'simple', 'tests']
+    >>> line2argv('a "more complex" test')
+    ['a', 'more complex', 'test']
+    >>> line2argv('a more="complex test of " quotes')
+    ['a', 'more=complex test of ', 'quotes']
+    >>> line2argv('a more" complex test of " quotes')
+    ['a', 'more complex test of ', 'quotes']
+    >>> line2argv('an "embedded \\"quote\\""')
+    ['an', 'embedded "quote"']
+
+    # Komodo bug 48027
+    >>> line2argv('foo bar C:\\')
+    ['foo', 'bar', 'C:\\']
+
+    # Komodo change 127581
+    >>> line2argv(r'"\test\slash" "foo bar" "foo\"bar"')
+    ['\\test\\slash', 'foo bar', 'foo"bar']
+
+    # Komodo change 127629
+    >>> if sys.platform == "win32":
+    ...     line2argv(r'\foo\bar') == ['\\foo\\bar']
+    ...     line2argv(r'\\foo\\bar') == ['\\\\foo\\\\bar']
+    ...     line2argv('"foo') == ['foo']
+    ... else:
+    ...     line2argv(r'\foo\bar') == ['foobar']
+    ...     line2argv(r'\\foo\\bar') == ['\\foo\\bar']
+    ...     try:
+    ...         line2argv('"foo')
+    ...     except ValueError, ex:
+    ...         "not terminated" in str(ex)
+    True
+    True
+    True
+    """
+    import string
+    line = line.strip()
+    argv = []
+    state = "default"
+    arg = None  # the current argument being parsed
+    i = -1
+    while 1:
+        i += 1
+        if i >= len(line): break
+        ch = line[i]
+
+        if ch == "\\" and i+1 < len(line):
+            # escaped char always added to arg, regardless of state
+            if arg is None: arg = ""
+            if (sys.platform == "win32"
+                or state in ("double-quoted", "single-quoted")
+               ) and line[i+1] not in tuple('"\''):
+                arg += ch
+            i += 1
+            arg += line[i]
+            continue
+
+        if state == "single-quoted":
+            if ch == "'":
+                state = "default"
+            else:
+                arg += ch
+        elif state == "double-quoted":
+            if ch == '"':
+                state = "default"
+            else:
+                arg += ch
+        elif state == "default":
+            if ch == '"':
+                if arg is None: arg = ""
+                state = "double-quoted"
+            elif ch == "'":
+                if arg is None: arg = ""
+                state = "single-quoted"
+            elif ch in string.whitespace:
+                if arg is not None:
+                    argv.append(arg)
+                arg = None
+            else:
+                if arg is None: arg = ""
+                arg += ch
+    if arg is not None:
+        argv.append(arg)
+    if not sys.platform == "win32" and state != "default":
+        raise ValueError("command line is not terminated: unfinished %s "
+                         "segment" % state)
+    return argv
+
+
+def argv2line(argv):
+    r"""Put together the given argument vector into a command line.
+    
+        "argv" is the argument vector to process.
+    
+    >>> from cmdln import argv2line
+    >>> argv2line(['foo'])
+    'foo'
+    >>> argv2line(['foo', 'bar'])
+    'foo bar'
+    >>> argv2line(['foo', 'bar baz'])
+    'foo "bar baz"'
+    >>> argv2line(['foo"bar'])
+    'foo"bar'
+    >>> print argv2line(['foo" bar'])
+    'foo" bar'
+    >>> print argv2line(["foo' bar"])
+    "foo' bar"
+    >>> argv2line(["foo'bar"])
+    "foo'bar"
+    """
+    escapedArgs = []
+    for arg in argv:
+        if ' ' in arg and '"' not in arg:
+            arg = '"'+arg+'"'
+        elif ' ' in arg and "'" not in arg:
+            arg = "'"+arg+"'"
+        elif ' ' in arg:
+            arg = arg.replace('"', r'\"')
+            arg = '"'+arg+'"'
+        escapedArgs.append(arg)
+    return ' '.join(escapedArgs)
+
+
+# Recipe: dedent (0.1) in /Users/trentm/tm/recipes/cookbook
+def _dedentlines(lines, tabsize=8, skip_first_line=False):
+    """_dedentlines(lines, tabsize=8, skip_first_line=False) -> dedented lines
+    
+        "lines" is a list of lines to dedent.
+        "tabsize" is the tab width to use for indent width calculations.
+        "skip_first_line" is a boolean indicating if the first line should
+            be skipped for calculating the indent width and for dedenting.
+            This is sometimes useful for docstrings and similar.
+    
+    Same as dedent() except operates on a sequence of lines. Note: the
+    lines list is modified **in-place**.
+    """
+    DEBUG = False
+    if DEBUG: 
+        print "dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\
+              % (tabsize, skip_first_line)
+    indents = []
+    margin = None
+    for i, line in enumerate(lines):
+        if i == 0 and skip_first_line: continue
+        indent = 0
+        for ch in line:
+            if ch == ' ':
+                indent += 1
+            elif ch == '\t':
+                indent += tabsize - (indent % tabsize)
+            elif ch in '\r\n':
+                continue # skip all-whitespace lines
+            else:
+                break
+        else:
+            continue # skip all-whitespace lines
+        if DEBUG: print "dedent: indent=%d: %r" % (indent, line)
+        if margin is None:
+            margin = indent
+        else:
+            margin = min(margin, indent)
+    if DEBUG: print "dedent: margin=%r" % margin
+
+    if margin is not None and margin > 0:
+        for i, line in enumerate(lines):
+            if i == 0 and skip_first_line: continue
+            removed = 0
+            for j, ch in enumerate(line):
+                if ch == ' ':
+                    removed += 1
+                elif ch == '\t':
+                    removed += tabsize - (removed % tabsize)
+                elif ch in '\r\n':
+                    if DEBUG: print "dedent: %r: EOL -> strip up to EOL" % line
+                    lines[i] = lines[i][j:]
+                    break
+                else:
+                    raise ValueError("unexpected non-whitespace char %r in "
+                                     "line %r while removing %d-space margin"
+                                     % (ch, line, margin))
+                if DEBUG:
+                    print "dedent: %r: %r -> removed %d/%d"\
+                          % (line, ch, removed, margin)
+                if removed == margin:
+                    lines[i] = lines[i][j+1:]
+                    break
+                elif removed > margin:
+                    lines[i] = ' '*(removed-margin) + lines[i][j+1:]
+                    break
+    return lines
+
+def _dedent(text, tabsize=8, skip_first_line=False):
+    """_dedent(text, tabsize=8, skip_first_line=False) -> dedented text
+
+        "text" is the text to dedent.
+        "tabsize" is the tab width to use for indent width calculations.
+        "skip_first_line" is a boolean indicating if the first line should
+            be skipped for calculating the indent width and for dedenting.
+            This is sometimes useful for docstrings and similar.
+    
+    textwrap.dedent(s), but don't expand tabs to spaces
+    """
+    lines = text.splitlines(1)
+    _dedentlines(lines, tabsize=tabsize, skip_first_line=skip_first_line)
+    return ''.join(lines)
+
+
+def _get_indent(marker, s, tab_width=8):
+    """_get_indent(marker, s, tab_width=8) ->
+        (<indentation-of-'marker'>, <indentation-width>)"""
+    # Figure out how much the marker is indented.
+    INDENT_CHARS = tuple(' \t')
+    start = s.index(marker)
+    i = start
+    while i > 0:
+        if s[i-1] not in INDENT_CHARS:
+            break
+        i -= 1
+    indent = s[i:start]
+    indent_width = 0
+    for ch in indent:
+        if ch == ' ':
+            indent_width += 1
+        elif ch == '\t':
+            indent_width += tab_width - (indent_width % tab_width)
+    return indent, indent_width
+
+def _get_trailing_whitespace(marker, s):
+    """Return the whitespace content trailing the given 'marker' in string 's',
+    up to and including a newline.
+    """
+    suffix = ''
+    start = s.index(marker) + len(marker)
+    i = start
+    while i < len(s):
+        if s[i] in ' \t':
+            suffix += s[i]
+        elif s[i] in '\r\n':
+            suffix += s[i]
+            if s[i] == '\r' and i+1 < len(s) and s[i+1] == '\n':
+                suffix += s[i+1]
+            break
+        else:
+            break
+        i += 1
+    return suffix
+
+
+
+#---- bash completion support
+# Note: This is still experimental. I expect to change this
+# significantly.
+#
+# To get Bash completion for a cmdln.Cmdln class, run the following
+# bash command:
+#   $ complete -C 'python -m cmdln /path/to/script.py CmdlnClass' cmdname
+# For example:
+#   $ complete -C 'python -m cmdln ~/bin/svn.py SVN' svn
+#
+#TODO: Simplify the above so don't have to given path to script (try to
+#      find it on PATH, if possible). Could also make class name
+#      optional if there is only one in the module (common case).
+
+if __name__ == "__main__" and len(sys.argv) == 6:
+    def _log(s):
+        return # no-op, comment out for debugging
+        from os.path import expanduser
+        fout = open(expanduser("~/tmp/bashcpln.log"), 'a')
+        fout.write(str(s) + '\n')
+        fout.close()
+
+    # Recipe: module_from_path (1.0.1+)
+    def _module_from_path(path):
+        import imp, os, sys
+        path = os.path.expanduser(path)
+        dir = os.path.dirname(path) or os.curdir
+        name = os.path.splitext(os.path.basename(path))[0]
+        sys.path.insert(0, dir)
+        try:
+            iinfo = imp.find_module(name, [dir])
+            return imp.load_module(name, *iinfo)
+        finally:
+            sys.path.remove(dir)
+
+    def _get_bash_cplns(script_path, class_name, cmd_name,
+                        token, preceding_token):
+        _log('--')
+        _log('get_cplns(%r, %r, %r, %r, %r)'
+             % (script_path, class_name, cmd_name, token, preceding_token))
+        comp_line = os.environ["COMP_LINE"]
+        comp_point = int(os.environ["COMP_POINT"])
+        _log("COMP_LINE: %r" % comp_line)
+        _log("COMP_POINT: %r" % comp_point)
+
+        try:
+            script = _module_from_path(script_path)
+        except ImportError, ex:
+            _log("error importing `%s': %s" % (script_path, ex))
+            return []
+        shell = getattr(script, class_name)()
+        cmd_map = shell._get_canonical_map()
+        del cmd_map["EOF"]
+
+        # Determine if completing the sub-command name.
+        parts = comp_line[:comp_point].split(None, 1)
+        _log(parts)
+        if len(parts) == 1 or not (' ' in parts[1] or '\t' in parts[1]):
+            #TODO: if parts[1].startswith('-'): handle top-level opts
+            _log("complete sub-command names")
+            matches = {}
+            for name, canon_name in cmd_map.items():
+                if name.startswith(token):
+                    matches[name] = canon_name
+            if not matches:
+                return []
+            elif len(matches) == 1:
+                return matches.keys()
+            elif len(set(matches.values())) == 1:
+                return [matches.values()[0]]
+            else:
+                return matches.keys()
+
+        # Otherwise, complete options for the given sub-command.
+        #TODO: refine this so it does the right thing with option args
+        if token.startswith('-'):
+            cmd_name = comp_line.split(None, 2)[1]
+            try:
+                cmd_canon_name = cmd_map[cmd_name]
+            except KeyError:
+                return []
+            handler = shell._get_cmd_handler(cmd_canon_name)
+            optparser = getattr(handler, "optparser", None)
+            if optparser is None:
+                optparser = SubCmdOptionParser()
+            opt_strs = []
+            for option in optparser.option_list:
+                for opt_str in option._short_opts + option._long_opts:
+                    if opt_str.startswith(token):
+                        opt_strs.append(opt_str)
+            return opt_strs
+
+        return []
+
+    for cpln in _get_bash_cplns(*sys.argv[1:]):
+        print cpln
+
diff --git a/scripts/lib/mic/utils/errors.py b/scripts/lib/mic/utils/errors.py
new file mode 100644
index 0000000000..8d720f9080
--- /dev/null
+++ b/scripts/lib/mic/utils/errors.py
@@ -0,0 +1,71 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2007 Red Hat, Inc.
+# Copyright (c) 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+class CreatorError(Exception):
+    """An exception base class for all imgcreate errors."""
+    keyword = '<creator>'
+
+    def __init__(self, msg):
+        self.msg = msg
+
+    def __str__(self):
+        if isinstance(self.msg, unicode):
+            self.msg = self.msg.encode('utf-8', 'ignore')
+        else:
+            self.msg = str(self.msg)
+        return self.keyword + self.msg
+
+class Usage(CreatorError):
+    keyword = '<usage>'
+
+    def __str__(self):
+        if isinstance(self.msg, unicode):
+            self.msg = self.msg.encode('utf-8', 'ignore')
+        else:
+            self.msg = str(self.msg)
+        return self.keyword + self.msg + ', please use "--help" for more info'
+
+class Abort(CreatorError):
+    keyword = ''
+
+class ConfigError(CreatorError):
+    keyword = '<config>'
+
+class KsError(CreatorError):
+    keyword = '<kickstart>'
+
+class RepoError(CreatorError):
+    keyword = '<repo>'
+
+class RpmError(CreatorError):
+    keyword = '<rpm>'
+
+class MountError(CreatorError):
+    keyword = '<mount>'
+
+class SnapshotError(CreatorError):
+    keyword = '<snapshot>'
+
+class SquashfsError(CreatorError):
+    keyword = '<squashfs>'
+
+class BootstrapError(CreatorError):
+    keyword = '<bootstrap>'
+
+class RuntimeError(CreatorError):
+    keyword = '<runtime>'
diff --git a/scripts/lib/mic/utils/fs_related.py b/scripts/lib/mic/utils/fs_related.py
new file mode 100644
index 0000000000..dd420e88dc
--- /dev/null
+++ b/scripts/lib/mic/utils/fs_related.py
@@ -0,0 +1,1060 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2007, Red Hat, Inc.
+# Copyright (c) 2009, 2010, 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+from __future__ import with_statement
+import os
+import sys
+import errno
+import stat
+import random
+import string
+import time
+import uuid
+
+from mic import msger
+from mic.utils import runner
+from mic.utils.errors import *
+from mic.utils.oe.misc import *
+
+def find_binary_inchroot(binary, chroot):
+    paths = ["/usr/sbin",
+             "/usr/bin",
+             "/sbin",
+             "/bin"
+            ]
+
+    for path in paths:
+        bin_path = "%s/%s" % (path, binary)
+        if os.path.exists("%s/%s" % (chroot, bin_path)):
+            return bin_path
+    return None
+
+def find_binary_path(binary):
+    if os.environ.has_key("PATH"):
+        paths = os.environ["PATH"].split(":")
+    else:
+        paths = []
+        if os.environ.has_key("HOME"):
+            paths += [os.environ["HOME"] + "/bin"]
+        paths += ["/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"]
+
+    for path in paths:
+        bin_path = "%s/%s" % (path, binary)
+        if os.path.exists(bin_path):
+            return bin_path
+
+    print "External command '%s' not found, exiting." % binary
+    print "  (Please install '%s' on your host system)" % binary
+    sys.exit(1)
+
+def makedirs(dirname):
+    """A version of os.makedirs() that doesn't throw an
+    exception if the leaf directory already exists.
+    """
+    try:
+        os.makedirs(dirname)
+    except OSError, err:
+        if err.errno != errno.EEXIST:
+            raise
+
+def mksquashfs(in_img, out_img):
+    fullpathmksquashfs = find_binary_path("mksquashfs")
+    args = [fullpathmksquashfs, in_img, out_img]
+
+    if not sys.stdout.isatty():
+        args.append("-no-progress")
+
+    ret = runner.show(args)
+    if ret != 0:
+        raise SquashfsError("'%s' exited with error (%d)" % (' '.join(args), ret))
+
+def resize2fs(fs, size):
+    resize2fs = find_binary_path("resize2fs")
+    if size == 0:
+        # it means to minimalize it
+        return runner.show([resize2fs, '-M', fs])
+    else:
+        return runner.show([resize2fs, fs, "%sK" % (size / 1024,)])
+
+def my_fuser(fp):
+    fuser = find_binary_path("fuser")
+    if not os.path.exists(fp):
+        return False
+
+    rc = runner.quiet([fuser, "-s", fp])
+    if rc == 0:
+        for pid in runner.outs([fuser, fp]).split():
+            fd = open("/proc/%s/cmdline" % pid, "r")
+            cmdline = fd.read()
+            fd.close()
+            if cmdline[:-1] == "/bin/bash":
+                return True
+
+    # not found
+    return False
+
+class BindChrootMount:
+    """Represents a bind mount of a directory into a chroot."""
+    def __init__(self, src, chroot, dest = None, option = None):
+        self.root = os.path.abspath(os.path.expanduser(chroot))
+        self.option = option
+
+        self.orig_src = self.src = src
+        if os.path.islink(src):
+            self.src = os.readlink(src)
+            if not self.src.startswith('/'):
+                self.src = os.path.abspath(os.path.join(os.path.dirname(src),
+                                                        self.src))
+
+        if not dest:
+            dest = self.src
+        self.dest = os.path.join(self.root, dest.lstrip('/'))
+
+        self.mounted = False
+        self.mountcmd = find_binary_path("mount")
+        self.umountcmd = find_binary_path("umount")
+
+    def ismounted(self):
+        with open('/proc/mounts') as f:
+            for line in f:
+                if line.split()[1] == os.path.abspath(self.dest):
+                    return True
+
+        return False
+
+    def has_chroot_instance(self):
+        lock = os.path.join(self.root, ".chroot.lock")
+        return my_fuser(lock)
+
+    def mount(self):
+        if self.mounted or self.ismounted():
+            return
+
+        makedirs(self.dest)
+        rc = runner.show([self.mountcmd, "--bind", self.src, self.dest])
+        if rc != 0:
+            raise MountError("Bind-mounting '%s' to '%s' failed" %
+                             (self.src, self.dest))
+        if self.option:
+            rc = runner.show([self.mountcmd, "--bind", "-o", "remount,%s" % self.option, self.dest])
+            if rc != 0:
+                raise MountError("Bind-remounting '%s' failed" % self.dest)
+
+        self.mounted = True
+        if os.path.islink(self.orig_src):
+            dest = os.path.join(self.root, self.orig_src.lstrip('/'))
+            if not os.path.exists(dest):
+                os.symlink(self.src, dest)
+
+    def unmount(self):
+        if self.has_chroot_instance():
+            return
+
+        if self.ismounted():
+            runner.show([self.umountcmd, "-l", self.dest])
+        self.mounted = False
+
+class LoopbackMount:
+    """LoopbackMount  compatibility layer for old API"""
+    def __init__(self, lofile, mountdir, fstype = None):
+        self.diskmount = DiskMount(LoopbackDisk(lofile,size = 0),mountdir,fstype,rmmountdir = True)
+        self.losetup = False
+        self.losetupcmd = find_binary_path("losetup")
+
+    def cleanup(self):
+        self.diskmount.cleanup()
+
+    def unmount(self):
+        self.diskmount.unmount()
+
+    def lounsetup(self):
+        if self.losetup:
+            runner.show([self.losetupcmd, "-d", self.loopdev])
+            self.losetup = False
+            self.loopdev = None
+
+    def loopsetup(self):
+        if self.losetup:
+            return
+
+        self.loopdev = get_loop_device(self.losetupcmd, self.lofile)
+        self.losetup = True
+
+    def mount(self):
+        self.diskmount.mount()
+
+class SparseLoopbackMount(LoopbackMount):
+    """SparseLoopbackMount  compatibility layer for old API"""
+    def __init__(self, lofile, mountdir, size, fstype = None):
+        self.diskmount = DiskMount(SparseLoopbackDisk(lofile,size),mountdir,fstype,rmmountdir = True)
+
+    def expand(self, create = False, size = None):
+        self.diskmount.disk.expand(create, size)
+
+    def truncate(self, size = None):
+        self.diskmount.disk.truncate(size)
+
+    def create(self):
+        self.diskmount.disk.create()
+
+class SparseExtLoopbackMount(SparseLoopbackMount):
+    """SparseExtLoopbackMount  compatibility layer for old API"""
+    def __init__(self, lofile, mountdir, size, fstype, blocksize, fslabel):
+        self.diskmount = ExtDiskMount(SparseLoopbackDisk(lofile,size), mountdir, fstype, blocksize, fslabel, rmmountdir = True)
+
+
+    def __format_filesystem(self):
+        self.diskmount.__format_filesystem()
+
+    def create(self):
+        self.diskmount.disk.create()
+
+    def resize(self, size = None):
+        return self.diskmount.__resize_filesystem(size)
+
+    def mount(self):
+        self.diskmount.mount()
+
+    def __fsck(self):
+        self.extdiskmount.__fsck()
+
+    def __get_size_from_filesystem(self):
+        return self.diskmount.__get_size_from_filesystem()
+
+    def __resize_to_minimal(self):
+        return self.diskmount.__resize_to_minimal()
+
+    def resparse(self, size = None):
+        return self.diskmount.resparse(size)
+
+class Disk:
+    """Generic base object for a disk
+
+    The 'create' method must make the disk visible as a block device - eg
+    by calling losetup. For RawDisk, this is obviously a no-op. The 'cleanup'
+    method must undo the 'create' operation.
+    """
+    def __init__(self, size, device = None):
+        self._device = device
+        self._size = size
+
+    def create(self):
+        pass
+
+    def cleanup(self):
+        pass
+
+    def get_device(self):
+        return self._device
+    def set_device(self, path):
+        self._device = path
+    device = property(get_device, set_device)
+
+    def get_size(self):
+        return self._size
+    size = property(get_size)
+
+
+class RawDisk(Disk):
+    """A Disk backed by a block device.
+    Note that create() is a no-op.
+    """
+    def __init__(self, size, device):
+        Disk.__init__(self, size, device)
+
+    def fixed(self):
+        return True
+
+    def exists(self):
+        return True
+
+
+class DiskImage(Disk):
+    """
+    A Disk backed by a file.
+    """
+    def __init__(self, image_file, size):
+        Disk.__init__(self, size)
+        self.image_file = image_file
+
+    def exists(self):
+        return os.path.exists(self.image_file)
+
+    def create(self):
+        if self.device is not None:
+            return
+
+        blocks = self.size / 1024
+        if self.size - blocks * 1024:
+            blocks += 1
+
+        # create disk image
+        dd_cmd = "dd if=/dev/zero of=%s bs=1024 seek=%d count=1" % \
+            (self.image_file, blocks)
+        rc, out = exec_cmd(dd_cmd)
+
+        self.device = self.image_file
+
+
+class LoopbackDisk(Disk):
+    """A Disk backed by a file via the loop module."""
+    def __init__(self, lofile, size):
+        Disk.__init__(self, size)
+        self.lofile = lofile
+        self.losetupcmd = find_binary_path("losetup")
+
+    def fixed(self):
+        return False
+
+    def exists(self):
+        return os.path.exists(self.lofile)
+
+    def create(self):
+        if self.device is not None:
+            return
+
+        self.device = get_loop_device(self.losetupcmd, self.lofile)
+
+    def cleanup(self):
+        if self.device is None:
+            return
+        msger.debug("Losetup remove %s" % self.device)
+        rc = runner.show([self.losetupcmd, "-d", self.device])
+        self.device = None
+
+class SparseLoopbackDisk(LoopbackDisk):
+    """A Disk backed by a sparse file via the loop module."""
+    def __init__(self, lofile, size):
+        LoopbackDisk.__init__(self, lofile, size)
+
+    def expand(self, create = False, size = None):
+        flags = os.O_WRONLY
+        if create:
+            flags |= os.O_CREAT
+            if not os.path.exists(self.lofile):
+                makedirs(os.path.dirname(self.lofile))
+
+        if size is None:
+            size = self.size
+
+        msger.debug("Extending sparse file %s to %d" % (self.lofile, size))
+        if create:
+            fd = os.open(self.lofile, flags, 0644)
+        else:
+            fd = os.open(self.lofile, flags)
+
+        if size <= 0:
+            size = 1
+        try:
+            os.ftruncate(fd, size)
+        except:
+            # may be limited by 2G in 32bit env
+            os.ftruncate(fd, 2**31L)
+
+        os.close(fd)
+
+    def truncate(self, size = None):
+        if size is None:
+            size = self.size
+
+        msger.debug("Truncating sparse file %s to %d" % (self.lofile, size))
+        fd = os.open(self.lofile, os.O_WRONLY)
+        os.ftruncate(fd, size)
+        os.close(fd)
+
+    def create(self):
+        self.expand(create = True)
+        LoopbackDisk.create(self)
+
+class Mount:
+    """A generic base class to deal with mounting things."""
+    def __init__(self, mountdir):
+        self.mountdir = mountdir
+
+    def cleanup(self):
+        self.unmount()
+
+    def mount(self, options = None):
+        pass
+
+    def unmount(self):
+        pass
+
+class DiskMount(Mount):
+    """A Mount object that handles mounting of a Disk."""
+    def __init__(self, disk, mountdir, fstype = None, rmmountdir = True):
+        Mount.__init__(self, mountdir)
+
+        self.disk = disk
+        self.fstype = fstype
+        self.rmmountdir = rmmountdir
+
+        self.mounted = False
+        self.rmdir   = False
+        if fstype:
+            self.mkfscmd = find_binary_path("mkfs." + self.fstype)
+        else:
+            self.mkfscmd = None
+        self.mountcmd = find_binary_path("mount")
+        self.umountcmd = find_binary_path("umount")
+
+    def cleanup(self):
+        Mount.cleanup(self)
+        self.disk.cleanup()
+
+    def unmount(self):
+        if self.mounted:
+            msger.debug("Unmounting directory %s" % self.mountdir)
+            runner.quiet('sync') # sync the data on this mount point
+            rc = runner.show([self.umountcmd, "-l", self.mountdir])
+            if rc == 0:
+                self.mounted = False
+            else:
+                raise MountError("Failed to umount %s" % self.mountdir)
+        if self.rmdir and not self.mounted:
+            try:
+                os.rmdir(self.mountdir)
+            except OSError, e:
+                pass
+            self.rmdir = False
+
+
+    def __create(self):
+        self.disk.create()
+
+
+    def mount(self, options = None):
+        if self.mounted:
+            return
+
+        if not os.path.isdir(self.mountdir):
+            msger.debug("Creating mount point %s" % self.mountdir)
+            os.makedirs(self.mountdir)
+            self.rmdir = self.rmmountdir
+
+        self.__create()
+
+        msger.debug("Mounting %s at %s" % (self.disk.device, self.mountdir))
+        if options:
+            args = [ self.mountcmd, "-o", options, self.disk.device, self.mountdir ]
+        else:
+            args = [ self.mountcmd, self.disk.device, self.mountdir ]
+        if self.fstype:
+            args.extend(["-t", self.fstype])
+
+        rc = runner.show(args)
+        if rc != 0:
+            raise MountError("Failed to mount '%s' to '%s' with command '%s'. Retval: %s" %
+                             (self.disk.device, self.mountdir, " ".join(args), rc))
+
+        self.mounted = True
+
+class ExtDiskMount(DiskMount):
+    """A DiskMount object that is able to format/resize ext[23] filesystems."""
+    def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None):
+        DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir)
+        self.blocksize = blocksize
+        self.fslabel = fslabel.replace("/", "")
+        self.uuid = str(uuid.uuid4())
+        self.skipformat = skipformat
+        self.fsopts = fsopts
+        self.extopts = None
+        self.dumpe2fs = find_binary_path("dumpe2fs")
+        self.tune2fs = find_binary_path("tune2fs")
+
+    def __parse_field(self, output, field):
+        for line in output.split("\n"):
+            if line.startswith(field + ":"):
+                return line[len(field) + 1:].strip()
+
+        raise KeyError("Failed to find field '%s' in output" % field)
+
+    def __format_filesystem(self):
+        if self.skipformat:
+            msger.debug("Skip filesystem format.")
+            return
+
+        msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device))
+        cmdlist = [self.mkfscmd, "-F", "-L", self.fslabel, "-m", "1", "-b",
+                   str(self.blocksize), "-U", self.uuid]
+        if self.extopts:
+            cmdlist.extend(self.extopts.split())
+        cmdlist.extend([self.disk.device])
+
+        rc, errout = runner.runtool(cmdlist, catch=2)
+        if rc != 0:
+            raise MountError("Error creating %s filesystem on disk %s:\n%s" %
+                             (self.fstype, self.disk.device, errout))
+
+        if not self.extopts:
+            msger.debug("Tuning filesystem on %s" % self.disk.device)
+            runner.show([self.tune2fs, "-c0", "-i0", "-Odir_index", "-ouser_xattr,acl", self.disk.device])
+
+    def __resize_filesystem(self, size = None):
+        current_size = os.stat(self.disk.lofile)[stat.ST_SIZE]
+
+        if size is None:
+            size = self.disk.size
+
+        if size == current_size:
+            return
+
+        if size > current_size:
+            self.disk.expand(size)
+
+        self.__fsck()
+
+        resize2fs(self.disk.lofile, size)
+        return size
+
+    def __create(self):
+        resize = False
+        if not self.disk.fixed() and self.disk.exists():
+            resize = True
+
+        self.disk.create()
+
+        if resize:
+            self.__resize_filesystem()
+        else:
+            self.__format_filesystem()
+
+    def mount(self, options = None):
+        self.__create()
+        DiskMount.mount(self, options)
+
+    def __fsck(self):
+        msger.info("Checking filesystem %s" % self.disk.lofile)
+        runner.quiet(["/sbin/e2fsck", "-f", "-y", self.disk.lofile])
+
+    def __get_size_from_filesystem(self):
+        return int(self.__parse_field(runner.outs([self.dumpe2fs, '-h', self.disk.lofile]),
+                                      "Block count")) * self.blocksize
+
+    def __resize_to_minimal(self):
+        self.__fsck()
+
+        #
+        # Use a binary search to find the minimal size
+        # we can resize the image to
+        #
+        bot = 0
+        top = self.__get_size_from_filesystem()
+        while top != (bot + 1):
+            t = bot + ((top - bot) / 2)
+
+            if not resize2fs(self.disk.lofile, t):
+                top = t
+            else:
+                bot = t
+        return top
+
+    def resparse(self, size = None):
+        self.cleanup()
+        if size == 0:
+            minsize = 0
+        else:
+            minsize = self.__resize_to_minimal()
+            self.disk.truncate(minsize)
+
+        self.__resize_filesystem(size)
+        return minsize
+
+class VfatDiskMount(DiskMount):
+    """A DiskMount object that is able to format vfat/msdos filesystems."""
+    def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None):
+        DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir)
+        self.blocksize = blocksize
+        self.fslabel = fslabel.replace("/", "")
+        rand1 = random.randint(0, 2**16 - 1)
+        rand2 = random.randint(0, 2**16 - 1)
+        self.uuid = "%04X-%04X" % (rand1, rand2)
+        self.skipformat = skipformat
+        self.fsopts = fsopts
+        self.fsckcmd = find_binary_path("fsck." + self.fstype)
+
+    def __format_filesystem(self):
+        if self.skipformat:
+            msger.debug("Skip filesystem format.")
+            return
+
+        msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device))
+        rc = runner.show([self.mkfscmd, "-n", self.fslabel,
+                          "-i", self.uuid.replace("-", ""), self.disk.device])
+        if rc != 0:
+            raise MountError("Error creating %s filesystem on disk %s" % (self.fstype,self.disk.device))
+
+        msger.verbose("Tuning filesystem on %s" % self.disk.device)
+
+    def __resize_filesystem(self, size = None):
+        current_size = os.stat(self.disk.lofile)[stat.ST_SIZE]
+
+        if size is None:
+            size = self.disk.size
+
+        if size == current_size:
+            return
+
+        if size > current_size:
+            self.disk.expand(size)
+
+        self.__fsck()
+
+        #resize2fs(self.disk.lofile, size)
+        return size
+
+    def __create(self):
+        resize = False
+        if not self.disk.fixed() and self.disk.exists():
+            resize = True
+
+        self.disk.create()
+
+        if resize:
+            self.__resize_filesystem()
+        else:
+            self.__format_filesystem()
+
+    def mount(self, options = None):
+        self.__create()
+        DiskMount.mount(self, options)
+
+    def __fsck(self):
+        msger.debug("Checking filesystem %s" % self.disk.lofile)
+        runner.show([self.fsckcmd, "-y", self.disk.lofile])
+
+    def __get_size_from_filesystem(self):
+        return self.disk.size
+
+    def __resize_to_minimal(self):
+        self.__fsck()
+
+        #
+        # Use a binary search to find the minimal size
+        # we can resize the image to
+        #
+        bot = 0
+        top = self.__get_size_from_filesystem()
+        return top
+
+    def resparse(self, size = None):
+        self.cleanup()
+        minsize = self.__resize_to_minimal()
+        self.disk.truncate(minsize)
+        self.__resize_filesystem(size)
+        return minsize
+
+class BtrfsDiskMount(DiskMount):
+    """A DiskMount object that is able to format/resize btrfs filesystems."""
+    def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None):
+        self.__check_btrfs()
+        DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir)
+        self.blocksize = blocksize
+        self.fslabel = fslabel.replace("/", "")
+        self.uuid  = None
+        self.skipformat = skipformat
+        self.fsopts = fsopts
+        self.blkidcmd = find_binary_path("blkid")
+        self.btrfsckcmd = find_binary_path("btrfsck")
+
+    def __check_btrfs(self):
+        found = False
+        """ Need to load btrfs module to mount it """
+        load_module("btrfs")
+        for line in open("/proc/filesystems").xreadlines():
+            if line.find("btrfs") > -1:
+                found = True
+                break
+        if not found:
+            raise MountError("Your system can't mount btrfs filesystem, please make sure your kernel has btrfs support and the module btrfs.ko has been loaded.")
+
+        # disable selinux, selinux will block write
+        if os.path.exists("/usr/sbin/setenforce"):
+            runner.show(["/usr/sbin/setenforce", "0"])
+
+    def __parse_field(self, output, field):
+        for line in output.split(" "):
+            if line.startswith(field + "="):
+                return line[len(field) + 1:].strip().replace("\"", "")
+
+        raise KeyError("Failed to find field '%s' in output" % field)
+
+    def __format_filesystem(self):
+        if self.skipformat:
+            msger.debug("Skip filesystem format.")
+            return
+
+        msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device))
+        rc = runner.show([self.mkfscmd, "-L", self.fslabel, self.disk.device])
+        if rc != 0:
+            raise MountError("Error creating %s filesystem on disk %s" % (self.fstype,self.disk.device))
+
+        self.uuid = self.__parse_field(runner.outs([self.blkidcmd, self.disk.device]), "UUID")
+
+    def __resize_filesystem(self, size = None):
+        current_size = os.stat(self.disk.lofile)[stat.ST_SIZE]
+
+        if size is None:
+            size = self.disk.size
+
+        if size == current_size:
+            return
+
+        if size > current_size:
+            self.disk.expand(size)
+
+        self.__fsck()
+        return size
+
+    def __create(self):
+        resize = False
+        if not self.disk.fixed() and self.disk.exists():
+            resize = True
+
+        self.disk.create()
+
+        if resize:
+            self.__resize_filesystem()
+        else:
+            self.__format_filesystem()
+
+    def mount(self, options = None):
+        self.__create()
+        DiskMount.mount(self, options)
+
+    def __fsck(self):
+        msger.debug("Checking filesystem %s" % self.disk.lofile)
+        runner.quiet([self.btrfsckcmd, self.disk.lofile])
+
+    def __get_size_from_filesystem(self):
+        return self.disk.size
+
+    def __resize_to_minimal(self):
+        self.__fsck()
+
+        return self.__get_size_from_filesystem()
+
+    def resparse(self, size = None):
+        self.cleanup()
+        minsize = self.__resize_to_minimal()
+        self.disk.truncate(minsize)
+        self.__resize_filesystem(size)
+        return minsize
+
+class DeviceMapperSnapshot(object):
+    def __init__(self, imgloop, cowloop):
+        self.imgloop = imgloop
+        self.cowloop = cowloop
+
+        self.__created = False
+        self.__name = None
+        self.dmsetupcmd = find_binary_path("dmsetup")
+
+        """Load dm_snapshot if it isn't loaded"""
+        load_module("dm_snapshot")
+
+    def get_path(self):
+        if self.__name is None:
+            return None
+        return os.path.join("/dev/mapper", self.__name)
+    path = property(get_path)
+
+    def create(self):
+        if self.__created:
+            return
+
+        self.imgloop.create()
+        self.cowloop.create()
+
+        self.__name = "imgcreate-%d-%d" % (os.getpid(),
+                                           random.randint(0, 2**16))
+
+        size = os.stat(self.imgloop.lofile)[stat.ST_SIZE]
+
+        table = "0 %d snapshot %s %s p 8" % (size / 512,
+                                             self.imgloop.device,
+                                             self.cowloop.device)
+
+        args = [self.dmsetupcmd, "create", self.__name, "--table", table]
+        if runner.show(args) != 0:
+            self.cowloop.cleanup()
+            self.imgloop.cleanup()
+            raise SnapshotError("Could not create snapshot device using: " + ' '.join(args))
+
+        self.__created = True
+
+    def remove(self, ignore_errors = False):
+        if not self.__created:
+            return
+
+        time.sleep(2)
+        rc = runner.show([self.dmsetupcmd, "remove", self.__name])
+        if not ignore_errors and rc != 0:
+            raise SnapshotError("Could not remove snapshot device")
+
+        self.__name = None
+        self.__created = False
+
+        self.cowloop.cleanup()
+        self.imgloop.cleanup()
+
+    def get_cow_used(self):
+        if not self.__created:
+            return 0
+
+        #
+        # dmsetup status on a snapshot returns e.g.
+        #   "0 8388608 snapshot 416/1048576"
+        # or, more generally:
+        #   "A B snapshot C/D"
+        # where C is the number of 512 byte sectors in use
+        #
+        out = runner.outs([self.dmsetupcmd, "status", self.__name])
+        try:
+            return int((out.split()[3]).split('/')[0]) * 512
+        except ValueError:
+            raise SnapshotError("Failed to parse dmsetup status: " + out)
+
+def create_image_minimizer(path, image, minimal_size):
+    """
+    Builds a copy-on-write image which can be used to
+    create a device-mapper snapshot of an image where
+    the image's filesystem is as small as possible
+
+    The steps taken are:
+      1) Create a sparse COW
+      2) Loopback mount the image and the COW
+      3) Create a device-mapper snapshot of the image
+         using the COW
+      4) Resize the filesystem to the minimal size
+      5) Determine the amount of space used in the COW
+      6) Restroy the device-mapper snapshot
+      7) Truncate the COW, removing unused space
+      8) Create a squashfs of the COW
+    """
+    imgloop = LoopbackDisk(image, None) # Passing bogus size - doesn't matter
+
+    cowloop = SparseLoopbackDisk(os.path.join(os.path.dirname(path), "osmin"),
+                                 64L * 1024L * 1024L)
+
+    snapshot = DeviceMapperSnapshot(imgloop, cowloop)
+
+    try:
+        snapshot.create()
+
+        resize2fs(snapshot.path, minimal_size)
+
+        cow_used = snapshot.get_cow_used()
+    finally:
+        snapshot.remove(ignore_errors = (not sys.exc_info()[0] is None))
+
+    cowloop.truncate(cow_used)
+
+    mksquashfs(cowloop.lofile, path)
+
+    os.unlink(cowloop.lofile)
+
+def load_module(module):
+    found = False
+    for line in open('/proc/modules').xreadlines():
+        if line.startswith("%s " % module):
+            found = True
+            break
+    if not found:
+        msger.info("Loading %s..." % module)
+        runner.quiet(['modprobe', module])
+
+class LoopDevice(object):
+    def __init__(self, loopid=None):
+        self.device = None
+        self.loopid = loopid
+        self.created = False
+        self.kpartxcmd = find_binary_path("kpartx")
+        self.losetupcmd = find_binary_path("losetup")
+
+    def register(self, device):
+        self.device = device
+        self.loopid = None
+        self.created = True
+
+    def reg_atexit(self):
+        import atexit
+        atexit.register(self.close)
+
+    def _genloopid(self):
+        import glob
+        if not glob.glob("/dev/loop[0-9]*"):
+            return 10
+
+        fint = lambda x: x[9:].isdigit() and int(x[9:]) or 0
+        maxid = 1 + max(filter(lambda x: x<100,
+                               map(fint, glob.glob("/dev/loop[0-9]*"))))
+        if maxid < 10: maxid = 10
+        if maxid >= 100: raise
+        return maxid
+
+    def _kpseek(self, device):
+        rc, out = runner.runtool([self.kpartxcmd, '-l', '-v', device])
+        if rc != 0:
+            raise MountError("Can't query dm snapshot on %s" % device)
+        for line in out.splitlines():
+            if line and line.startswith("loop"):
+                return True
+        return False
+
+    def _loseek(self, device):
+        import re
+        rc, out = runner.runtool([self.losetupcmd, '-a'])
+        if rc != 0:
+            raise MountError("Failed to run 'losetup -a'")
+        for line in out.splitlines():
+            m = re.match("([^:]+): .*", line)
+            if m and m.group(1) == device:
+                return True
+        return False
+
+    def create(self):
+        if not self.created:
+            if not self.loopid:
+                self.loopid = self._genloopid()
+            self.device = "/dev/loop%d" % self.loopid
+            if os.path.exists(self.device):
+                if self._loseek(self.device):
+                    raise MountError("Device busy: %s" % self.device)
+                else:
+                    self.created = True
+                    return
+
+            mknod = find_binary_path('mknod')
+            rc = runner.show([mknod, '-m664', self.device, 'b', '7', str(self.loopid)])
+            if rc != 0:
+                raise MountError("Failed to create device %s" % self.device)
+            else:
+                self.created = True
+
+    def close(self):
+        if self.created:
+            try:
+                self.cleanup()
+                self.device = None
+            except MountError, e:
+                msger.error("%s" % e)
+
+    def cleanup(self):
+
+        if self.device is None:
+            return
+
+
+        if self._kpseek(self.device):
+            if self.created:
+                for i in range(3, os.sysconf("SC_OPEN_MAX")):
+                    try:
+                        os.close(i)
+                    except:
+                        pass
+            runner.quiet([self.kpartxcmd, "-d", self.device])
+        if self._loseek(self.device):
+            runner.quiet([self.losetupcmd, "-d", self.device])
+        # FIXME: should sleep a while between two loseek
+        if self._loseek(self.device):
+            msger.warning("Can't cleanup loop device %s" % self.device)
+        elif self.loopid:
+            os.unlink(self.device)
+
+DEVICE_PIDFILE_DIR = "/var/tmp/mic/device"
+DEVICE_LOCKFILE = "/var/lock/__mic_loopdev.lock"
+
+def get_loop_device(losetupcmd, lofile):
+    global DEVICE_PIDFILE_DIR
+    global DEVICE_LOCKFILE
+
+    import fcntl
+    makedirs(os.path.dirname(DEVICE_LOCKFILE))
+    fp = open(DEVICE_LOCKFILE, 'w')
+    fcntl.flock(fp, fcntl.LOCK_EX)
+    try:
+        loopdev = None
+        devinst = LoopDevice()
+
+        # clean up left loop device first
+        clean_loop_devices()
+
+        # provide an avaible loop device
+        rc, out = runner.runtool([losetupcmd, "--find"])
+        if rc == 0:
+            loopdev = out.split()[0]
+            devinst.register(loopdev)
+        if not loopdev or not os.path.exists(loopdev):
+            devinst.create()
+            loopdev = devinst.device
+
+        # setup a loop device for image file
+        rc = runner.show([losetupcmd, loopdev, lofile])
+        if rc != 0:
+            raise MountError("Failed to setup loop device for '%s'" % lofile)
+
+        devinst.reg_atexit()
+
+        # try to save device and pid
+        makedirs(DEVICE_PIDFILE_DIR)
+        pidfile = os.path.join(DEVICE_PIDFILE_DIR, os.path.basename(loopdev))
+        if os.path.exists(pidfile):
+            os.unlink(pidfile)
+        with open(pidfile, 'w') as wf:
+            wf.write(str(os.getpid()))
+
+    except MountError, err:
+        raise CreatorError("%s" % str(err))
+    except:
+        raise
+    finally:
+        try:
+            fcntl.flock(fp, fcntl.LOCK_UN)
+            fp.close()
+            os.unlink(DEVICE_LOCKFILE)
+        except:
+            pass
+
+    return loopdev
+
+def clean_loop_devices(piddir=DEVICE_PIDFILE_DIR):
+    if not os.path.exists(piddir) or not os.path.isdir(piddir):
+        return
+
+    for loopdev in os.listdir(piddir):
+        pidfile = os.path.join(piddir, loopdev)
+        try:
+            with open(pidfile, 'r') as rf:
+                devpid = int(rf.read())
+        except:
+            devpid = None
+
+        # if the process using this device is alive, skip it
+        if not devpid or os.path.exists(os.path.join('/proc', str(devpid))):
+            continue
+
+        # try to clean it up
+        try:
+            devinst = LoopDevice()
+            devinst.register(os.path.join('/dev', loopdev))
+            devinst.cleanup()
+            os.unlink(pidfile)
+        except:
+            pass
+
diff --git a/scripts/lib/mic/utils/gpt_parser.py b/scripts/lib/mic/utils/gpt_parser.py
new file mode 100644
index 0000000000..5d43b70778
--- /dev/null
+++ b/scripts/lib/mic/utils/gpt_parser.py
@@ -0,0 +1,331 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2013 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+""" This module implements a simple GPT partitions parser which can read the
+GPT header and the GPT partition table. """
+
+import struct
+import uuid
+import binascii
+from mic.utils.errors import MountError
+
+_GPT_HEADER_FORMAT = "<8s4sIIIQQQQ16sQIII"
+_GPT_HEADER_SIZE = struct.calcsize(_GPT_HEADER_FORMAT)
+_GPT_ENTRY_FORMAT = "<16s16sQQQ72s"
+_GPT_ENTRY_SIZE = struct.calcsize(_GPT_ENTRY_FORMAT)
+_SUPPORTED_GPT_REVISION = '\x00\x00\x01\x00'
+
+def _stringify_uuid(binary_uuid):
+    """ A small helper function to transform a binary UUID into a string
+    format. """
+
+    uuid_str = str(uuid.UUID(bytes_le = binary_uuid))
+
+    return uuid_str.upper()
+
+def _calc_header_crc(raw_hdr):
+    """ Calculate GPT header CRC32 checksum. The 'raw_hdr' parameter has to
+    be a list or a tuple containing all the elements of the GPT header in a
+    "raw" form, meaning that it should simply contain "unpacked" disk data.
+    """
+
+    raw_hdr = list(raw_hdr)
+    raw_hdr[3] = 0
+    raw_hdr = struct.pack(_GPT_HEADER_FORMAT, *raw_hdr)
+
+    return binascii.crc32(raw_hdr) & 0xFFFFFFFF
+
+def _validate_header(raw_hdr):
+    """ Validate the GPT header. The 'raw_hdr' parameter has to be a list or a
+    tuple containing all the elements of the GPT header in a "raw" form,
+    meaning that it should simply contain "unpacked" disk data. """
+
+    # Validate the signature
+    if raw_hdr[0] != 'EFI PART':
+        raise MountError("GPT partition table not found")
+
+    # Validate the revision
+    if raw_hdr[1] != _SUPPORTED_GPT_REVISION:
+        raise MountError("Unsupported GPT revision '%s', supported revision " \
+                         "is '%s'" % \
+                          (binascii.hexlify(raw_hdr[1]),
+                           binascii.hexlify(_SUPPORTED_GPT_REVISION)))
+
+    # Validate header size
+    if raw_hdr[2] != _GPT_HEADER_SIZE:
+        raise MountError("Bad GPT header size: %d bytes, expected %d" % \
+                         (raw_hdr[2], _GPT_HEADER_SIZE))
+
+    crc = _calc_header_crc(raw_hdr)
+    if raw_hdr[3] != crc:
+        raise MountError("GPT header crc mismatch: %#x, should be %#x" % \
+                         (crc, raw_hdr[3]))
+
+class GptParser:
+    """ GPT partition table parser. Allows reading the GPT header and the
+    partition table, as well as modifying the partition table records. """
+
+    def __init__(self, disk_path, sector_size = 512):
+        """ The class constructor which accepts the following parameters:
+            * disk_path - full path to the disk image or device node
+            * sector_size - size of a disk sector in bytes """
+
+        self.sector_size = sector_size
+        self.disk_path = disk_path
+
+        try:
+            self._disk_obj = open(disk_path, 'r+b')
+        except IOError as err:
+            raise MountError("Cannot open file '%s' for reading GPT " \
+                             "partitions: %s" % (disk_path, err))
+
+    def __del__(self):
+        """ The class destructor. """
+
+        self._disk_obj.close()
+
+    def _read_disk(self, offset, size):
+        """ A helper function which reads 'size' bytes from offset 'offset' of
+        the disk and checks all the error conditions. """
+
+        self._disk_obj.seek(offset)
+        try:
+            data = self._disk_obj.read(size)
+        except IOError as err:
+            raise MountError("cannot read from '%s': %s" % \
+                             (self.disk_path, err))
+
+        if len(data) != size:
+            raise MountError("cannot read %d bytes from offset '%d' of '%s', " \
+                             "read only %d bytes" % \
+                             (size, offset, self.disk_path, len(data)))
+
+        return data
+
+    def _write_disk(self, offset, buf):
+        """ A helper function which writes buffer 'buf' to offset 'offset' of
+        the disk. This function takes care of unaligned writes and checks all
+        the error conditions. """
+
+        # Since we may be dealing with a block device, we only can write in
+        # 'self.sector_size' chunks. Find the aligned starting and ending
+        # disk offsets to read.
+        start =  (offset / self.sector_size) * self.sector_size
+        end = ((start + len(buf)) / self.sector_size + 1) * self.sector_size
+
+        data = self._read_disk(start, end - start)
+        off = offset - start
+        data = data[:off] + buf + data[off + len(buf):]
+
+        self._disk_obj.seek(start)
+        try:
+            self._disk_obj.write(data)
+        except IOError as err:
+            raise MountError("cannot write to '%s': %s" % (self.disk_path, err))
+
+    def read_header(self, primary = True):
+        """ Read and verify the GPT header and return a dictionary containing
+        the following elements:
+
+        'signature'   : header signature
+        'revision'    : header revision
+        'hdr_size'    : header size in bytes
+        'hdr_crc'     : header CRC32
+        'hdr_lba'     : LBA of this header
+        'hdr_offs'    : byte disk offset of this header
+        'backup_lba'  : backup header LBA
+        'backup_offs' : byte disk offset of backup header
+        'first_lba'   : first usable LBA for partitions
+        'first_offs'  : first usable byte disk offset for partitions
+        'last_lba'    : last usable LBA for partitions
+        'last_offs'   : last usable byte disk offset for partitions
+        'disk_uuid'   : UUID of the disk
+        'ptable_lba'  : starting LBA of array of partition entries
+        'ptable_offs' : disk byte offset of the start of the partition table
+        'ptable_size' : partition table size in bytes
+        'entries_cnt' : number of available partition table entries
+        'entry_size'  : size of a single partition entry
+        'ptable_crc'  : CRC32 of the partition table
+        'primary'     : a boolean, if 'True', this is the primary GPT header,
+                        if 'False' - the secondary
+        'primary_str' : contains string "primary" if this is the primary GPT
+                        header, and "backup" otherwise
+
+        This dictionary corresponds to the GPT header format. Please, see the
+        UEFI standard for the description of these fields.
+
+        If the 'primary' parameter is 'True', the primary GPT header is read,
+        otherwise the backup GPT header is read instead. """
+
+        # Read and validate the primary GPT header
+        raw_hdr = self._read_disk(self.sector_size, _GPT_HEADER_SIZE)
+        raw_hdr = struct.unpack(_GPT_HEADER_FORMAT, raw_hdr)
+        _validate_header(raw_hdr)
+        primary_str = "primary"
+
+        if not primary:
+            # Read and validate the backup GPT header
+            raw_hdr = self._read_disk(raw_hdr[6] * self.sector_size, _GPT_HEADER_SIZE)
+            raw_hdr = struct.unpack(_GPT_HEADER_FORMAT, raw_hdr)
+            _validate_header(raw_hdr)
+            primary_str = "backup"
+
+        return { 'signature'   : raw_hdr[0],
+                 'revision'    : raw_hdr[1],
+                 'hdr_size'    : raw_hdr[2],
+                 'hdr_crc'     : raw_hdr[3],
+                 'hdr_lba'     : raw_hdr[5],
+                 'hdr_offs'    : raw_hdr[5] * self.sector_size,
+                 'backup_lba'  : raw_hdr[6],
+                 'backup_offs' : raw_hdr[6] * self.sector_size,
+                 'first_lba'   : raw_hdr[7],
+                 'first_offs'  : raw_hdr[7] * self.sector_size,
+                 'last_lba'    : raw_hdr[8],
+                 'last_offs'   : raw_hdr[8] * self.sector_size,
+                 'disk_uuid'   :_stringify_uuid(raw_hdr[9]),
+                 'ptable_lba'  : raw_hdr[10],
+                 'ptable_offs' : raw_hdr[10] * self.sector_size,
+                 'ptable_size' : raw_hdr[11] * raw_hdr[12],
+                 'entries_cnt' : raw_hdr[11],
+                 'entry_size'  : raw_hdr[12],
+                 'ptable_crc'  : raw_hdr[13],
+                 'primary'     : primary,
+                 'primary_str' : primary_str }
+
+    def _read_raw_ptable(self, header):
+        """ Read and validate primary or backup partition table. The 'header'
+        argument is the GPT header. If it is the primary GPT header, then the
+        primary partition table is read and validated, otherwise - the backup
+        one. The 'header' argument is a dictionary which is returned by the
+        'read_header()' method. """
+
+        raw_ptable = self._read_disk(header['ptable_offs'],
+                                     header['ptable_size'])
+
+        crc = binascii.crc32(raw_ptable) & 0xFFFFFFFF
+        if crc != header['ptable_crc']:
+            raise MountError("Partition table at LBA %d (%s) is corrupted" % \
+                             (header['ptable_lba'], header['primary_str']))
+
+        return raw_ptable
+
+    def get_partitions(self, primary = True):
+        """ This is a generator which parses the GPT partition table and
+        generates the following dictionary for each partition:
+
+        'index'       : the index of the partition table endry
+        'offs'        : byte disk offset of the partition table entry
+        'type_uuid'   : partition type UUID
+        'part_uuid'   : partition UUID
+        'first_lba'   : the first LBA
+        'last_lba'    : the last LBA
+        'flags'       : attribute flags
+        'name'        : partition name
+        'primary'     : a boolean, if 'True', this is the primary partition
+                        table, if 'False' - the secondary
+        'primary_str' : contains string "primary" if this is the primary GPT
+                        header, and "backup" otherwise
+
+        This dictionary corresponds to the GPT header format. Please, see the
+        UEFI standard for the description of these fields.
+
+        If the 'primary' parameter is 'True', partitions from the primary GPT
+        partition table are generated, otherwise partitions from the backup GPT
+        partition table are generated. """
+
+        if primary:
+            primary_str = "primary"
+        else:
+            primary_str = "backup"
+
+        header = self.read_header(primary)
+        raw_ptable = self._read_raw_ptable(header)
+
+        for index in xrange(0, header['entries_cnt']):
+            start = header['entry_size'] * index
+            end = start + header['entry_size']
+            raw_entry = struct.unpack(_GPT_ENTRY_FORMAT, raw_ptable[start:end])
+
+            if raw_entry[2] == 0 or raw_entry[3] == 0:
+                continue
+
+            part_name = str(raw_entry[5].decode('UTF-16').split('\0', 1)[0])
+
+            yield { 'index'       : index,
+                    'offs'        : header['ptable_offs'] + start,
+                    'type_uuid'   : _stringify_uuid(raw_entry[0]),
+                    'part_uuid'   : _stringify_uuid(raw_entry[1]),
+                    'first_lba'   : raw_entry[2],
+                    'last_lba'    : raw_entry[3],
+                    'flags'       : raw_entry[4],
+                    'name'        : part_name,
+                    'primary'     : primary,
+                    'primary_str' : primary_str }
+
+    def _change_partition(self, header, entry):
+        """ A helper function for 'change_partitions()' which changes a
+        a paricular instance of the partition table (primary or backup). """
+
+        if entry['index'] >= header['entries_cnt']:
+            raise MountError("Partition table at LBA %d has only %d "   \
+                             "records cannot change record number %d" % \
+                             (header['entries_cnt'], entry['index']))
+        # Read raw GPT header
+        raw_hdr = self._read_disk(header['hdr_offs'], _GPT_HEADER_SIZE)
+        raw_hdr = list(struct.unpack(_GPT_HEADER_FORMAT, raw_hdr))
+        _validate_header(raw_hdr)
+
+        # Prepare the new partition table entry
+        raw_entry = struct.pack(_GPT_ENTRY_FORMAT,
+                                uuid.UUID(entry['type_uuid']).bytes_le,
+                                uuid.UUID(entry['part_uuid']).bytes_le,
+                                entry['first_lba'],
+                                entry['last_lba'],
+                                entry['flags'],
+                                entry['name'].encode('UTF-16'))
+
+        # Write the updated entry to the disk
+        entry_offs = header['ptable_offs'] + \
+                     header['entry_size'] * entry['index']
+        self._write_disk(entry_offs, raw_entry)
+
+        # Calculate and update partition table CRC32
+        raw_ptable = self._read_disk(header['ptable_offs'],
+                                     header['ptable_size'])
+        raw_hdr[13] = binascii.crc32(raw_ptable) & 0xFFFFFFFF
+
+        # Calculate and update the GPT header CRC
+        raw_hdr[3] = _calc_header_crc(raw_hdr)
+
+        # Write the updated header to the disk
+        raw_hdr = struct.pack(_GPT_HEADER_FORMAT, *raw_hdr)
+        self._write_disk(header['hdr_offs'], raw_hdr)
+
+    def change_partition(self, entry):
+        """ Change a GPT partition. The 'entry' argument has the same format as
+        'get_partitions()' returns. This function simply changes the partition
+        table record corresponding to 'entry' in both, the primary and the
+        backup GPT partition tables. The parition table CRC is re-calculated
+        and the GPT headers are modified accordingly. """
+
+        # Change the primary partition table
+        header = self.read_header(True)
+        self._change_partition(header, entry)
+
+        # Change the backup partition table
+        header = self.read_header(False)
+        self._change_partition(header, entry)
diff --git a/scripts/lib/mic/utils/grabber.py b/scripts/lib/mic/utils/grabber.py
new file mode 100644
index 0000000000..45e30b4fb0
--- /dev/null
+++ b/scripts/lib/mic/utils/grabber.py
@@ -0,0 +1,97 @@
+#!/usr/bin/python
+
+import os
+import sys
+import rpm
+import fcntl
+import struct
+import termios
+
+from mic import msger
+from mic.utils import runner
+from mic.utils.errors import CreatorError
+
+from urlgrabber import grabber
+from urlgrabber import __version__ as grabber_version
+
+if rpm.labelCompare(grabber_version.split('.'), '3.9.0'.split('.')) == -1:
+    msger.warning("Version of python-urlgrabber is %s, lower than '3.9.0', "
+                  "you may encounter some network issues" % grabber_version)
+
+def myurlgrab(url, filename, proxies, progress_obj = None):
+    g = grabber.URLGrabber()
+    if progress_obj is None:
+        progress_obj = TextProgress()
+
+    if url.startswith("file:/"):
+        filepath = "/%s" % url.replace("file:", "").lstrip('/')
+        if not os.path.exists(filepath):
+            raise CreatorError("URLGrabber error: can't find file %s" % url)
+        if url.endswith('.rpm'):
+            return filepath
+        else:
+            # untouch repometadata in source path
+            runner.show(['cp', '-f', filepath, filename])
+
+    else:
+        try:
+            filename = g.urlgrab(url=str(url),
+                                 filename=filename,
+                                 ssl_verify_host=False,
+                                 ssl_verify_peer=False,
+                                 proxies=proxies,
+                                 http_headers=(('Pragma', 'no-cache'),),
+                                 quote=0,
+                                 progress_obj=progress_obj)
+        except grabber.URLGrabError, err:
+            msg = str(err)
+            if msg.find(url) < 0:
+                msg += ' on %s' % url
+            raise CreatorError(msg)
+
+    return filename
+
+def terminal_width(fd=1):
+    """ Get the real terminal width """
+    try:
+        buf = 'abcdefgh'
+        buf = fcntl.ioctl(fd, termios.TIOCGWINSZ, buf)
+        return struct.unpack('hhhh', buf)[1]
+    except: # IOError
+        return 80
+
+def truncate_url(url, width):
+    return os.path.basename(url)[0:width]
+
+class TextProgress(object):
+    # make the class as singleton
+    _instance = None
+    def __new__(cls, *args, **kwargs):
+        if not cls._instance:
+            cls._instance = super(TextProgress, cls).__new__(cls, *args, **kwargs)
+
+        return cls._instance
+
+    def __init__(self, totalnum = None):
+        self.total = totalnum
+        self.counter = 1
+
+    def start(self, filename, url, *args, **kwargs):
+        self.url = url
+        self.termwidth = terminal_width()
+        msger.info("\r%-*s" % (self.termwidth, " "))
+        if self.total is None:
+            msger.info("\rRetrieving %s ..." % truncate_url(self.url, self.termwidth - 15))
+        else:
+            msger.info("\rRetrieving %s [%d/%d] ..." % (truncate_url(self.url, self.termwidth - 25), self.counter, self.total))
+
+    def update(self, *args):
+        pass
+
+    def end(self, *args):
+        if self.counter == self.total:
+            msger.raw("\n")
+
+        if self.total is not None:
+            self.counter += 1
+
diff --git a/scripts/lib/mic/utils/misc.py b/scripts/lib/mic/utils/misc.py
new file mode 100644
index 0000000000..95241d7f15
--- /dev/null
+++ b/scripts/lib/mic/utils/misc.py
@@ -0,0 +1,1065 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2010, 2011 Intel Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os
+import sys
+import time
+import tempfile
+import re
+import shutil
+import glob
+import hashlib
+import subprocess
+import platform
+import traceback
+
+
+try:
+    import sqlite3 as sqlite
+except ImportError:
+    import sqlite
+
+try:
+    from xml.etree import cElementTree
+except ImportError:
+    import cElementTree
+xmlparse = cElementTree.parse
+
+from mic import msger
+from mic.utils.errors import CreatorError, SquashfsError
+from mic.utils.fs_related import find_binary_path, makedirs
+from mic.utils.proxy import get_proxy_for
+from mic.utils import runner
+
+
+RPM_RE  = re.compile("(.*)\.(.*) (.*)-(.*)")
+RPM_FMT = "%(name)s.%(arch)s %(version)s-%(release)s"
+SRPM_RE = re.compile("(.*)-(\d+.*)-(\d+\.\d+).src.rpm")
+
+
+def build_name(kscfg, release=None, prefix = None, suffix = None):
+    """Construct and return an image name string.
+
+    This is a utility function to help create sensible name and fslabel
+    strings. The name is constructed using the sans-prefix-and-extension
+    kickstart filename and the supplied prefix and suffix.
+
+    kscfg -- a path to a kickstart file
+    release --  a replacement to suffix for image release
+    prefix -- a prefix to prepend to the name; defaults to None, which causes
+              no prefix to be used
+    suffix -- a suffix to append to the name; defaults to None, which causes
+              a YYYYMMDDHHMM suffix to be used
+
+    Note, if maxlen is less then the len(suffix), you get to keep both pieces.
+
+    """
+    name = os.path.basename(kscfg)
+    idx = name.rfind('.')
+    if idx >= 0:
+        name = name[:idx]
+
+    if release is not None:
+        suffix = ""
+    if prefix is None:
+        prefix = ""
+    if suffix is None:
+        suffix = time.strftime("%Y%m%d%H%M")
+
+    if name.startswith(prefix):
+        name = name[len(prefix):]
+
+    prefix = "%s-" % prefix if prefix else ""
+    suffix = "-%s" % suffix if suffix else ""
+
+    ret = prefix + name + suffix
+    return ret
+
+def get_distro():
+    """Detect linux distribution, support "meego"
+    """
+
+    support_dists = ('SuSE',
+                     'debian',
+                     'fedora',
+                     'redhat',
+                     'centos',
+                     'meego',
+                     'moblin',
+                     'tizen')
+    try:
+        (dist, ver, id) = platform.linux_distribution( \
+                              supported_dists = support_dists)
+    except:
+        (dist, ver, id) = platform.dist( \
+                              supported_dists = support_dists)
+
+    return (dist, ver, id)
+
+def get_distro_str():
+    """Get composited string for current linux distribution
+    """
+    (dist, ver, id) = get_distro()
+
+    if not dist:
+        return 'Unknown Linux Distro'
+    else:
+        distro_str = ' '.join(map(str.strip, (dist, ver, id)))
+        return distro_str.strip()
+
+_LOOP_RULE_PTH = None
+
+def hide_loopdev_presentation():
+    udev_rules = "80-prevent-loop-present.rules"
+    udev_rules_dir = [
+                       '/usr/lib/udev/rules.d/',
+                       '/lib/udev/rules.d/',
+                       '/etc/udev/rules.d/'
+                     ]
+
+    global _LOOP_RULE_PTH
+
+    for rdir in udev_rules_dir:
+        if os.path.exists(rdir):
+            _LOOP_RULE_PTH = os.path.join(rdir, udev_rules)
+
+    if not _LOOP_RULE_PTH:
+        return
+
+    try:
+        with open(_LOOP_RULE_PTH, 'w') as wf:
+            wf.write('KERNEL=="loop*", ENV{UDISKS_PRESENTATION_HIDE}="1"')
+
+        runner.quiet('udevadm trigger')
+    except:
+        pass
+
+def unhide_loopdev_presentation():
+    global _LOOP_RULE_PTH
+
+    if not _LOOP_RULE_PTH:
+        return
+
+    try:
+        os.unlink(_LOOP_RULE_PTH)
+        runner.quiet('udevadm trigger')
+    except:
+        pass
+
+def extract_rpm(rpmfile, targetdir):
+    rpm2cpio = find_binary_path("rpm2cpio")
+    cpio = find_binary_path("cpio")
+
+    olddir = os.getcwd()
+    os.chdir(targetdir)
+
+    msger.verbose("Extract rpm file with cpio: %s" % rpmfile)
+    p1 = subprocess.Popen([rpm2cpio, rpmfile], stdout=subprocess.PIPE)
+    p2 = subprocess.Popen([cpio, "-idv"], stdin=p1.stdout,
+                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    (sout, serr) = p2.communicate()
+    msger.verbose(sout or serr)
+
+    os.chdir(olddir)
+
+def compressing(fpath, method):
+    comp_map = {
+        "gz": "gzip",
+        "bz2": "bzip2"
+    }
+    if method not in comp_map:
+        raise CreatorError("Unsupport compress format: %s, valid values: %s"
+                           % (method, ','.join(comp_map.keys())))
+    cmd = find_binary_path(comp_map[method])
+    rc = runner.show([cmd, "-f", fpath])
+    if rc:
+        raise CreatorError("Failed to %s file: %s" % (comp_map[method], fpath))
+
+def taring(dstfile, target):
+    import tarfile
+    basen, ext = os.path.splitext(dstfile)
+    comp = {".tar": None,
+            ".gz": "gz", # for .tar.gz
+            ".bz2": "bz2", # for .tar.bz2
+            ".tgz": "gz",
+            ".tbz": "bz2"}[ext]
+
+    # specify tarball file path
+    if not comp:
+        tarpath = dstfile
+    elif basen.endswith(".tar"):
+        tarpath = basen
+    else:
+        tarpath = basen + ".tar"
+    wf = tarfile.open(tarpath, 'w')
+
+    if os.path.isdir(target):
+        for item in os.listdir(target):
+            wf.add(os.path.join(target, item), item)
+    else:
+        wf.add(target, os.path.basename(target))
+    wf.close()
+
+    if comp:
+        compressing(tarpath, comp)
+        # when dstfile ext is ".tgz" and ".tbz", should rename
+        if not basen.endswith(".tar"):
+            shutil.move("%s.%s" % (tarpath, comp), dstfile)
+
+def ziping(dstfile, target):
+    import zipfile
+    wf = zipfile.ZipFile(dstfile, 'w', compression=zipfile.ZIP_DEFLATED)
+    if os.path.isdir(target):
+        for item in os.listdir(target):
+            fpath = os.path.join(target, item)
+            if not os.path.isfile(fpath):
+                continue
+            wf.write(fpath, item, zipfile.ZIP_DEFLATED)
+    else:
+        wf.write(target, os.path.basename(target), zipfile.ZIP_DEFLATED)
+    wf.close()
+
+pack_formats = {
+    ".tar": taring,
+    ".tar.gz": taring,
+    ".tar.bz2": taring,
+    ".tgz": taring,
+    ".tbz": taring,
+    ".zip": ziping,
+}
+
+def packing(dstfile, target):
+    (base, ext) = os.path.splitext(dstfile)
+    if ext in (".gz", ".bz2") and base.endswith(".tar"):
+        ext = ".tar" + ext
+    if ext not in pack_formats:
+        raise CreatorError("Unsupport pack format: %s, valid values: %s"
+                           % (ext, ','.join(pack_formats.keys())))
+    func = pack_formats[ext]
+    # func should be callable
+    func(dstfile, target)
+
+def human_size(size):
+    """Return human readable string for Bytes size
+    """
+
+    if size <= 0:
+        return "0M"
+    import math
+    measure = ['B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']
+    expo = int(math.log(size, 1024))
+    mant = float(size/math.pow(1024, expo))
+    return "{0:.1f}{1:s}".format(mant, measure[expo])
+
+def get_block_size(file_obj):
+    """ Returns block size for file object 'file_obj'. Errors are indicated by
+    the 'IOError' exception. """
+
+    from fcntl import ioctl
+    import struct
+
+    # Get the block size of the host file-system for the image file by calling
+    # the FIGETBSZ ioctl (number 2).
+    binary_data = ioctl(file_obj, 2, struct.pack('I', 0))
+    return struct.unpack('I', binary_data)[0]
+
+def check_space_pre_cp(src, dst):
+    """Check whether disk space is enough before 'cp' like
+    operations, else exception will be raised.
+    """
+
+    srcsize  = get_file_size(src) * 1024 * 1024
+    freesize = get_filesystem_avail(dst)
+    if srcsize > freesize:
+        raise CreatorError("space on %s(%s) is not enough for about %s files"
+                           % (dst, human_size(freesize), human_size(srcsize)))
+
+def calc_hashes(file_path, hash_names, start = 0, end = None):
+    """ Calculate hashes for a file. The 'file_path' argument is the file
+    to calculate hash functions for, 'start' and 'end' are the starting and
+    ending file offset to calculate the has functions for. The 'hash_names'
+    argument is a list of hash names to calculate. Returns the the list
+    of calculated hash values in the hexadecimal form in the same order
+    as 'hash_names'.
+    """
+    if end == None:
+        end = os.path.getsize(file_path)
+
+    chunk_size = 65536
+    to_read = end - start
+    read = 0
+
+    hashes = []
+    for hash_name in hash_names:
+        hashes.append(hashlib.new(hash_name))
+
+    with open(file_path, "rb") as f:
+        f.seek(start)
+
+        while read < to_read:
+            if read + chunk_size > to_read:
+                chunk_size = to_read - read
+            chunk = f.read(chunk_size)
+            for hash_obj in hashes:
+                hash_obj.update(chunk)
+            read += chunk_size
+
+    result = []
+    for hash_obj in hashes:
+        result.append(hash_obj.hexdigest())
+
+    return result
+
+def get_md5sum(fpath):
+    return calc_hashes(fpath, ('md5', ))[0]
+
+
+def normalize_ksfile(ksconf, release, arch):
+    '''
+    Return the name of a normalized ks file in which macro variables
+    @BUILD_ID@ and @ARCH@ are replace with real values.
+
+    The original ks file is returned if no special macro is used, otherwise
+    a temp file is created and returned, which will be deleted when program
+    exits normally.
+    '''
+
+    if not release:
+        release = "latest"
+    if not arch or re.match(r'i.86', arch):
+        arch = "ia32"
+
+    with open(ksconf) as f:
+        ksc = f.read()
+
+    if "@ARCH@" not in ksc and "@BUILD_ID@" not in ksc:
+        return ksconf
+
+    msger.info("Substitute macro variable @BUILD_ID@/@ARCH@ in ks: %s" % ksconf)
+    ksc = ksc.replace("@ARCH@", arch)
+    ksc = ksc.replace("@BUILD_ID@", release)
+
+    fd, ksconf = tempfile.mkstemp(prefix=os.path.basename(ksconf))
+    os.write(fd, ksc)
+    os.close(fd)
+
+    msger.debug('normalized ks file:%s' % ksconf)
+
+    def remove_temp_ks():
+        try:
+            os.unlink(ksconf)
+        except OSError, err:
+            msger.warning('Failed to remove temp ks file:%s:%s' % (ksconf, err))
+
+    import atexit
+    atexit.register(remove_temp_ks)
+
+    return ksconf
+
+
+def _check_mic_chroot(rootdir):
+    def _path(path):
+        return rootdir.rstrip('/') + path
+
+    release_files = map(_path, [ "/etc/moblin-release",
+                                 "/etc/meego-release",
+                                 "/etc/tizen-release"])
+
+    if not any(map(os.path.exists, release_files)):
+        msger.warning("Dir %s is not a MeeGo/Tizen chroot env" % rootdir)
+
+    if not glob.glob(rootdir + "/boot/vmlinuz-*"):
+        msger.warning("Failed to find kernel module under %s" % rootdir)
+
+    return
+
+def selinux_check(arch, fstypes):
+    try:
+        getenforce = find_binary_path('getenforce')
+    except CreatorError:
+        return
+
+    selinux_status = runner.outs([getenforce])
+    if arch and arch.startswith("arm") and selinux_status == "Enforcing":
+        raise CreatorError("Can't create arm image if selinux is enabled, "
+                           "please run 'setenforce 0' to disable selinux")
+
+    use_btrfs = filter(lambda typ: typ == 'btrfs', fstypes)
+    if use_btrfs and selinux_status == "Enforcing":
+        raise CreatorError("Can't create btrfs image if selinux is enabled,"
+                           " please run 'setenforce 0' to disable selinux")
+
+def get_image_type(path):
+    def _get_extension_name(path):
+        match = re.search("(?<=\.)\w+$", path)
+        if match:
+            return match.group(0)
+        else:
+            return None
+
+    if os.path.isdir(path):
+        _check_mic_chroot(path)
+        return "fs"
+
+    maptab = {
+              "tar": "loop",
+              "raw":"raw",
+              "vmdk":"vmdk",
+              "vdi":"vdi",
+              "iso":"livecd",
+              "usbimg":"liveusb",
+             }
+
+    extension = _get_extension_name(path)
+    if extension in maptab:
+        return maptab[extension]
+
+    fd = open(path, "rb")
+    file_header = fd.read(1024)
+    fd.close()
+    vdi_flag = "<<< Sun VirtualBox Disk Image >>>"
+    if file_header[0:len(vdi_flag)] == vdi_flag:
+        return maptab["vdi"]
+
+    output = runner.outs(['file', path])
+    isoptn = re.compile(r".*ISO 9660 CD-ROM filesystem.*(bootable).*")
+    usbimgptn = re.compile(r".*x86 boot sector.*active.*")
+    rawptn = re.compile(r".*x86 boot sector.*")
+    vmdkptn = re.compile(r".*VMware. disk image.*")
+    ext3fsimgptn = re.compile(r".*Linux.*ext3 filesystem data.*")
+    ext4fsimgptn = re.compile(r".*Linux.*ext4 filesystem data.*")
+    btrfsimgptn = re.compile(r".*BTRFS.*")
+    if isoptn.match(output):
+        return maptab["iso"]
+    elif usbimgptn.match(output):
+        return maptab["usbimg"]
+    elif rawptn.match(output):
+        return maptab["raw"]
+    elif vmdkptn.match(output):
+        return maptab["vmdk"]
+    elif ext3fsimgptn.match(output):
+        return "ext3fsimg"
+    elif ext4fsimgptn.match(output):
+        return "ext4fsimg"
+    elif btrfsimgptn.match(output):
+        return "btrfsimg"
+    else:
+        raise CreatorError("Cannot detect the type of image: %s" % path)
+
+
+def get_file_size(filename):
+    """ Return size in MB unit """
+    cmd = ['du', "-s", "-b", "-B", "1M", filename]
+    rc, duOutput  = runner.runtool(cmd)
+    if rc != 0:
+        raise CreatorError("Failed to run: %s" % ' '.join(cmd))
+    size1 = int(duOutput.split()[0])
+
+    cmd = ['du', "-s", "-B", "1M", filename]
+    rc, duOutput = runner.runtool(cmd)
+    if rc != 0:
+        raise CreatorError("Failed to run: %s" % ' '.join(cmd))
+
+    size2 = int(duOutput.split()[0])
+    return max(size1, size2)
+
+
+def get_filesystem_avail(fs):
+    vfstat = os.statvfs(fs)
+    return vfstat.f_bavail * vfstat.f_bsize
+
+def convert_image(srcimg, srcfmt, dstimg, dstfmt):
+    #convert disk format
+    if dstfmt != "raw":
+        raise CreatorError("Invalid destination image format: %s" % dstfmt)
+    msger.debug("converting %s image to %s" % (srcimg, dstimg))
+    if srcfmt == "vmdk":
+        path = find_binary_path("qemu-img")
+        argv = [path, "convert", "-f", "vmdk", srcimg, "-O", dstfmt,  dstimg]
+    elif srcfmt == "vdi":
+        path = find_binary_path("VBoxManage")
+        argv = [path, "internalcommands", "converttoraw", srcimg, dstimg]
+    else:
+        raise CreatorError("Invalid soure image format: %s" % srcfmt)
+
+    rc = runner.show(argv)
+    if rc == 0:
+        msger.debug("convert successful")
+    if rc != 0:
+        raise CreatorError("Unable to convert disk to %s" % dstfmt)
+
+def uncompress_squashfs(squashfsimg, outdir):
+    """Uncompress file system from squshfs image"""
+    unsquashfs = find_binary_path("unsquashfs")
+    args = [ unsquashfs, "-d", outdir, squashfsimg ]
+    rc = runner.show(args)
+    if (rc != 0):
+        raise SquashfsError("Failed to uncompress %s." % squashfsimg)
+
+def mkdtemp(dir = "/var/tmp", prefix = "wic-tmp-"):
+    """ FIXME: use the dir in wic.conf instead """
+
+    makedirs(dir)
+    return tempfile.mkdtemp(dir = dir, prefix = prefix)
+
+def get_repostrs_from_ks(ks):
+    def _get_temp_reponame(baseurl):
+        md5obj = hashlib.md5(baseurl)
+        tmpreponame = "%s" % md5obj.hexdigest()
+        return tmpreponame
+
+    kickstart_repos = []
+
+    for repodata in ks.handler.repo.repoList:
+        repo = {}
+        for attr in ('name',
+                     'baseurl',
+                     'mirrorlist',
+                     'includepkgs', # val is list
+                     'excludepkgs', # val is list
+                     'cost',    # int
+                     'priority',# int
+                     'save',
+                     'proxy',
+                     'proxyuser',
+                     'proxypasswd',
+                     'proxypasswd',
+                     'debuginfo',
+                     'source',
+                     'gpgkey',
+                     'ssl_verify'):
+            if hasattr(repodata, attr) and getattr(repodata, attr):
+                repo[attr] = getattr(repodata, attr)
+
+        if 'name' not in repo:
+            repo['name'] = _get_temp_reponame(repodata.baseurl)
+
+        kickstart_repos.append(repo)
+
+    return kickstart_repos
+
+def _get_uncompressed_data_from_url(url, filename, proxies):
+    filename = myurlgrab(url, filename, proxies)
+    suffix = None
+    if filename.endswith(".gz"):
+        suffix = ".gz"
+        runner.quiet(['gunzip', "-f", filename])
+    elif filename.endswith(".bz2"):
+        suffix = ".bz2"
+        runner.quiet(['bunzip2', "-f", filename])
+    if suffix:
+        filename = filename.replace(suffix, "")
+    return filename
+
+def _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, filename,
+                            sumtype=None, checksum=None):
+    url = os.path.join(baseurl, filename)
+    filename_tmp = str("%s/%s/%s" % (cachedir, reponame, os.path.basename(filename)))
+    if os.path.splitext(filename_tmp)[1] in (".gz", ".bz2"):
+        filename = os.path.splitext(filename_tmp)[0]
+    else:
+        filename = filename_tmp
+    if sumtype and checksum and os.path.exists(filename):
+        try:
+            sumcmd = find_binary_path("%ssum" % sumtype)
+        except:
+            file_checksum = None
+        else:
+            file_checksum = runner.outs([sumcmd, filename]).split()[0]
+
+        if file_checksum and file_checksum == checksum:
+            return filename
+
+    return _get_uncompressed_data_from_url(url,filename_tmp,proxies)
+
+def get_metadata_from_repos(repos, cachedir):
+    my_repo_metadata = []
+    for repo in repos:
+        reponame = repo['name']
+        baseurl  = repo['baseurl']
+
+
+        if 'proxy' in repo:
+            proxy = repo['proxy']
+        else:
+            proxy = get_proxy_for(baseurl)
+
+        proxies = None
+        if proxy:
+            proxies = {str(baseurl.split(":")[0]):str(proxy)}
+
+        makedirs(os.path.join(cachedir, reponame))
+        url = os.path.join(baseurl, "repodata/repomd.xml")
+        filename = os.path.join(cachedir, reponame, 'repomd.xml')
+        repomd = myurlgrab(url, filename, proxies)
+        try:
+            root = xmlparse(repomd)
+        except SyntaxError:
+            raise CreatorError("repomd.xml syntax error.")
+
+        ns = root.getroot().tag
+        ns = ns[0:ns.rindex("}")+1]
+
+        filepaths = {}
+        checksums = {}
+        sumtypes = {}
+
+        for elm in root.getiterator("%sdata" % ns):
+            if elm.attrib["type"] == "patterns":
+                filepaths['patterns'] = elm.find("%slocation" % ns).attrib['href']
+                checksums['patterns'] = elm.find("%sopen-checksum" % ns).text
+                sumtypes['patterns'] = elm.find("%sopen-checksum" % ns).attrib['type']
+                break
+
+        for elm in root.getiterator("%sdata" % ns):
+            if elm.attrib["type"] in ("group_gz", "group"):
+                filepaths['comps'] = elm.find("%slocation" % ns).attrib['href']
+                checksums['comps'] = elm.find("%sopen-checksum" % ns).text
+                sumtypes['comps'] = elm.find("%sopen-checksum" % ns).attrib['type']
+                break
+
+        primary_type = None
+        for elm in root.getiterator("%sdata" % ns):
+            if elm.attrib["type"] in ("primary_db", "primary"):
+                primary_type = elm.attrib["type"]
+                filepaths['primary'] = elm.find("%slocation" % ns).attrib['href']
+                checksums['primary'] = elm.find("%sopen-checksum" % ns).text
+                sumtypes['primary'] = elm.find("%sopen-checksum" % ns).attrib['type']
+                break
+
+        if not primary_type:
+            continue
+
+        for item in ("primary", "patterns", "comps"):
+            if item not in filepaths:
+                filepaths[item] = None
+                continue
+            if not filepaths[item]:
+                continue
+            filepaths[item] = _get_metadata_from_repo(baseurl,
+                                                      proxies,
+                                                      cachedir,
+                                                      reponame,
+                                                      filepaths[item],
+                                                      sumtypes[item],
+                                                      checksums[item])
+
+        """ Get repo key """
+        try:
+            repokey = _get_metadata_from_repo(baseurl,
+                                              proxies,
+                                              cachedir,
+                                              reponame,
+                                              "repodata/repomd.xml.key")
+        except CreatorError:
+            repokey = None
+            msger.debug("\ncan't get %s/%s" % (baseurl, "repodata/repomd.xml.key"))
+
+        my_repo_metadata.append({"name":reponame,
+                                 "baseurl":baseurl,
+                                 "repomd":repomd,
+                                 "primary":filepaths['primary'],
+                                 "cachedir":cachedir,
+                                 "proxies":proxies,
+                                 "patterns":filepaths['patterns'],
+                                 "comps":filepaths['comps'],
+                                 "repokey":repokey})
+
+    return my_repo_metadata
+
+def get_rpmver_in_repo(repometadata):
+    for repo in repometadata:
+        if repo["primary"].endswith(".xml"):
+            root = xmlparse(repo["primary"])
+            ns = root.getroot().tag
+            ns = ns[0:ns.rindex("}")+1]
+
+            versionlist = []
+            for elm in root.getiterator("%spackage" % ns):
+                if elm.find("%sname" % ns).text == 'rpm':
+                    for node in elm.getchildren():
+                        if node.tag == "%sversion" % ns:
+                            versionlist.append(node.attrib['ver'])
+
+            if versionlist:
+                return reversed(
+                         sorted(
+                           versionlist,
+                           key = lambda ver: map(int, ver.split('.')))).next()
+
+        elif repo["primary"].endswith(".sqlite"):
+            con = sqlite.connect(repo["primary"])
+            for row in con.execute("select version from packages where "
+                                   "name=\"rpm\" ORDER by version DESC"):
+                con.close()
+                return row[0]
+
+    return None
+
+def get_arch(repometadata):
+    archlist = []
+    for repo in repometadata:
+        if repo["primary"].endswith(".xml"):
+            root = xmlparse(repo["primary"])
+            ns = root.getroot().tag
+            ns = ns[0:ns.rindex("}")+1]
+            for elm in root.getiterator("%spackage" % ns):
+                if elm.find("%sarch" % ns).text not in ("noarch", "src"):
+                    arch = elm.find("%sarch" % ns).text
+                    if arch not in archlist:
+                        archlist.append(arch)
+        elif repo["primary"].endswith(".sqlite"):
+            con = sqlite.connect(repo["primary"])
+            for row in con.execute("select arch from packages where arch not in (\"src\", \"noarch\")"):
+                if row[0] not in archlist:
+                    archlist.append(row[0])
+
+            con.close()
+
+    uniq_arch = []
+    for i in range(len(archlist)):
+        if archlist[i] not in rpmmisc.archPolicies.keys():
+            continue
+        need_append = True
+        j = 0
+        while j < len(uniq_arch):
+            if archlist[i] in rpmmisc.archPolicies[uniq_arch[j]].split(':'):
+                need_append = False
+                break
+            if uniq_arch[j] in rpmmisc.archPolicies[archlist[i]].split(':'):
+                if need_append:
+                    uniq_arch[j] = archlist[i]
+                    need_append = False
+                else:
+                    uniq_arch.remove(uniq_arch[j])
+                    continue
+            j += 1
+        if need_append:
+             uniq_arch.append(archlist[i])
+
+    return uniq_arch, archlist
+
+def get_package(pkg, repometadata, arch = None):
+    ver = ""
+    target_repo = None
+    if not arch:
+        arches = []
+    elif arch not in rpmmisc.archPolicies:
+        arches = [arch]
+    else:
+        arches = rpmmisc.archPolicies[arch].split(':')
+        arches.append('noarch')
+
+    for repo in repometadata:
+        if repo["primary"].endswith(".xml"):
+            root = xmlparse(repo["primary"])
+            ns = root.getroot().tag
+            ns = ns[0:ns.rindex("}")+1]
+            for elm in root.getiterator("%spackage" % ns):
+                if elm.find("%sname" % ns).text == pkg:
+                    if elm.find("%sarch" % ns).text in arches:
+                        version = elm.find("%sversion" % ns)
+                        tmpver = "%s-%s" % (version.attrib['ver'], version.attrib['rel'])
+                        if tmpver > ver:
+                            ver = tmpver
+                            location = elm.find("%slocation" % ns)
+                            pkgpath = "%s" % location.attrib['href']
+                            target_repo = repo
+                        break
+        if repo["primary"].endswith(".sqlite"):
+            con = sqlite.connect(repo["primary"])
+            if arch:
+                sql = 'select version, release, location_href from packages ' \
+                      'where name = "%s" and arch IN ("%s")' % \
+                      (pkg, '","'.join(arches))
+                for row in con.execute(sql):
+                    tmpver = "%s-%s" % (row[0], row[1])
+                    if tmpver > ver:
+                        ver = tmpver
+                        pkgpath = "%s" % row[2]
+                        target_repo = repo
+                    break
+            else:
+                sql = 'select version, release, location_href from packages ' \
+                      'where name = "%s"' % pkg
+                for row in con.execute(sql):
+                    tmpver = "%s-%s" % (row[0], row[1])
+                    if tmpver > ver:
+                        ver = tmpver
+                        pkgpath = "%s" % row[2]
+                        target_repo = repo
+                    break
+            con.close()
+    if target_repo:
+        makedirs("%s/packages/%s" % (target_repo["cachedir"], target_repo["name"]))
+        url = os.path.join(target_repo["baseurl"], pkgpath)
+        filename = str("%s/packages/%s/%s" % (target_repo["cachedir"], target_repo["name"], os.path.basename(pkgpath)))
+        if os.path.exists(filename):
+            ret = rpmmisc.checkRpmIntegrity('rpm', filename)
+            if ret == 0:
+                return filename
+
+            msger.warning("package %s is damaged: %s" %
+                          (os.path.basename(filename), filename))
+            os.unlink(filename)
+
+        pkg = myurlgrab(str(url), filename, target_repo["proxies"])
+        return pkg
+    else:
+        return None
+
+def get_source_name(pkg, repometadata):
+
+    def get_bin_name(pkg):
+        m = RPM_RE.match(pkg)
+        if m:
+            return m.group(1)
+        return None
+
+    def get_src_name(srpm):
+        m = SRPM_RE.match(srpm)
+        if m:
+            return m.group(1)
+        return None
+
+    ver = ""
+    target_repo = None
+
+    pkg_name = get_bin_name(pkg)
+    if not pkg_name:
+        return None
+
+    for repo in repometadata:
+        if repo["primary"].endswith(".xml"):
+            root = xmlparse(repo["primary"])
+            ns = root.getroot().tag
+            ns = ns[0:ns.rindex("}")+1]
+            for elm in root.getiterator("%spackage" % ns):
+                if elm.find("%sname" % ns).text == pkg_name:
+                    if elm.find("%sarch" % ns).text != "src":
+                        version = elm.find("%sversion" % ns)
+                        tmpver = "%s-%s" % (version.attrib['ver'], version.attrib['rel'])
+                        if tmpver > ver:
+                            ver = tmpver
+                            fmt = elm.find("%sformat" % ns)
+                            if fmt:
+                                fns = fmt.getchildren()[0].tag
+                                fns = fns[0:fns.rindex("}")+1]
+                                pkgpath = fmt.find("%ssourcerpm" % fns).text
+                                target_repo = repo
+                        break
+
+        if repo["primary"].endswith(".sqlite"):
+            con = sqlite.connect(repo["primary"])
+            for row in con.execute("select version, release, rpm_sourcerpm from packages where name = \"%s\" and arch != \"src\"" % pkg_name):
+                tmpver = "%s-%s" % (row[0], row[1])
+                if tmpver > ver:
+                    pkgpath = "%s" % row[2]
+                    target_repo = repo
+                break
+            con.close()
+    if target_repo:
+        return get_src_name(pkgpath)
+    else:
+        return None
+
+def get_pkglist_in_patterns(group, patterns):
+    found = False
+    pkglist = []
+    try:
+        root = xmlparse(patterns)
+    except SyntaxError:
+        raise SyntaxError("%s syntax error." % patterns)
+
+    for elm in list(root.getroot()):
+        ns = elm.tag
+        ns = ns[0:ns.rindex("}")+1]
+        name = elm.find("%sname" % ns)
+        summary = elm.find("%ssummary" % ns)
+        if name.text == group or summary.text == group:
+            found = True
+            break
+
+    if not found:
+        return pkglist
+
+    found = False
+    for requires in list(elm):
+        if requires.tag.endswith("requires"):
+            found = True
+            break
+
+    if not found:
+        return pkglist
+
+    for pkg in list(requires):
+        pkgname = pkg.attrib["name"]
+        if pkgname not in pkglist:
+            pkglist.append(pkgname)
+
+    return pkglist
+
+def get_pkglist_in_comps(group, comps):
+    found = False
+    pkglist = []
+    try:
+        root = xmlparse(comps)
+    except SyntaxError:
+        raise SyntaxError("%s syntax error." % comps)
+
+    for elm in root.getiterator("group"):
+        id = elm.find("id")
+        name = elm.find("name")
+        if id.text == group or name.text == group:
+            packagelist = elm.find("packagelist")
+            found = True
+            break
+
+    if not found:
+        return pkglist
+
+    for require in elm.getiterator("packagereq"):
+        if require.tag.endswith("packagereq"):
+            pkgname = require.text
+        if pkgname not in pkglist:
+            pkglist.append(pkgname)
+
+    return pkglist
+
+def is_statically_linked(binary):
+    return ", statically linked, " in runner.outs(['file', binary])
+
+def setup_qemu_emulator(rootdir, arch):
+    # mount binfmt_misc if it doesn't exist
+    if not os.path.exists("/proc/sys/fs/binfmt_misc"):
+        modprobecmd = find_binary_path("modprobe")
+        runner.show([modprobecmd, "binfmt_misc"])
+    if not os.path.exists("/proc/sys/fs/binfmt_misc/register"):
+        mountcmd = find_binary_path("mount")
+        runner.show([mountcmd, "-t", "binfmt_misc", "none", "/proc/sys/fs/binfmt_misc"])
+
+    # qemu_emulator is a special case, we can't use find_binary_path
+    # qemu emulator should be a statically-linked executable file
+    qemu_emulator = "/usr/bin/qemu-arm"
+    if not os.path.exists(qemu_emulator) or not is_statically_linked(qemu_emulator):
+        qemu_emulator = "/usr/bin/qemu-arm-static"
+    if not os.path.exists(qemu_emulator):
+        raise CreatorError("Please install a statically-linked qemu-arm")
+
+    # qemu emulator version check
+    armv7_list = [arch for arch in rpmmisc.archPolicies.keys() if arch.startswith('armv7')]
+    if arch in armv7_list:  # need qemu (>=0.13.0)
+        qemuout = runner.outs([qemu_emulator, "-h"])
+        m = re.search("version\s*([.\d]+)", qemuout)
+        if m:
+            qemu_version = m.group(1)
+            if qemu_version < "0.13":
+                raise CreatorError("Requires %s version >=0.13 for %s" % (qemu_emulator, arch))
+        else:
+            msger.warning("Can't get version info of %s, please make sure it's higher than 0.13.0" % qemu_emulator)
+
+    if not os.path.exists(rootdir + "/usr/bin"):
+        makedirs(rootdir + "/usr/bin")
+    shutil.copy(qemu_emulator, rootdir + "/usr/bin/qemu-arm-static")
+    qemu_emulator = "/usr/bin/qemu-arm-static"
+
+    # disable selinux, selinux will block qemu emulator to run
+    if os.path.exists("/usr/sbin/setenforce"):
+        msger.info('Try to disable selinux')
+        runner.show(["/usr/sbin/setenforce", "0"])
+
+    # unregister it if it has been registered and is a dynamically-linked executable
+    node = "/proc/sys/fs/binfmt_misc/arm"
+    if os.path.exists(node):
+        qemu_unregister_string = "-1\n"
+        fd = open("/proc/sys/fs/binfmt_misc/arm", "w")
+        fd.write(qemu_unregister_string)
+        fd.close()
+
+    # register qemu emulator for interpreting other arch executable file
+    if not os.path.exists(node):
+        qemu_arm_string = ":arm:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfa\\xff\\xff\\xff:%s:\n" % qemu_emulator
+        fd = open("/proc/sys/fs/binfmt_misc/register", "w")
+        fd.write(qemu_arm_string)
+        fd.close()
+
+    return qemu_emulator
+
+def SrcpkgsDownload(pkgs, repometadata, instroot, cachedir):
+    def get_source_repometadata(repometadata):
+        src_repometadata=[]
+        for repo in repometadata:
+            if repo["name"].endswith("-source"):
+                src_repometadata.append(repo)
+        if src_repometadata:
+            return src_repometadata
+        return None
+
+    def get_src_name(srpm):
+        m = SRPM_RE.match(srpm)
+        if m:
+            return m.group(1)
+        return None
+
+    src_repometadata = get_source_repometadata(repometadata)
+
+    if not src_repometadata:
+        msger.warning("No source repo found")
+        return None
+
+    src_pkgs = []
+    lpkgs_dict = {}
+    lpkgs_path = []
+    for repo in src_repometadata:
+        cachepath = "%s/%s/packages/*.src.rpm" %(cachedir, repo["name"])
+        lpkgs_path += glob.glob(cachepath)
+
+    for lpkg in lpkgs_path:
+        lpkg_name = get_src_name(os.path.basename(lpkg))
+        lpkgs_dict[lpkg_name] = lpkg
+    localpkgs = lpkgs_dict.keys()
+
+    cached_count = 0
+    destdir = instroot+'/usr/src/SRPMS'
+    if not os.path.exists(destdir):
+        os.makedirs(destdir)
+
+    srcpkgset = set()
+    for _pkg in pkgs:
+        srcpkg_name = get_source_name(_pkg, repometadata)
+        if not srcpkg_name:
+            continue
+        srcpkgset.add(srcpkg_name)
+
+    for pkg in list(srcpkgset):
+        if pkg in localpkgs:
+            cached_count += 1
+            shutil.copy(lpkgs_dict[pkg], destdir)
+            src_pkgs.append(os.path.basename(lpkgs_dict[pkg]))
+        else:
+            src_pkg = get_package(pkg, src_repometadata, 'src')
+            if src_pkg:
+                shutil.copy(src_pkg, destdir)
+                src_pkgs.append(src_pkg)
+    msger.info("%d source packages gotten from cache" % cached_count)
+
+    return src_pkgs
+
+def strip_end(text, suffix):
+    if not text.endswith(suffix):
+        return text
+    return text[:-len(suffix)]
diff --git a/scripts/lib/mic/utils/oe/__init__.py b/scripts/lib/mic/utils/oe/__init__.py
new file mode 100644
index 0000000000..d10e802116
--- /dev/null
+++ b/scripts/lib/mic/utils/oe/__init__.py
@@ -0,0 +1,22 @@
+#
+# OpenEmbedded mic utils library
+#
+# Copyright (c) 2013, Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] linux.intel.com>
+#
diff --git a/scripts/lib/mic/utils/oe/misc.py b/scripts/lib/mic/utils/oe/misc.py
new file mode 100644
index 0000000000..7ad3aa9685
--- /dev/null
+++ b/scripts/lib/mic/utils/oe/misc.py
@@ -0,0 +1,144 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (c) 2013, Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION
+# This module provides a place to collect various mic-related utils
+# for the OpenEmbedded Image Tools.
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] linux.intel.com>
+#
+
+from mic import msger
+from mic.utils import runner
+
+def exec_cmd(cmd_and_args, as_shell = False, catch = 3):
+    """
+    Execute command, catching stderr, stdout
+
+    Need to execute as_shell if the command uses wildcards
+    """
+    msger.debug("exec_cmd: %s" % cmd_and_args)
+    args = cmd_and_args.split()
+    msger.debug(args)
+
+    if (as_shell):
+        rc, out = runner.runtool(cmd_and_args, catch)
+    else:
+        rc, out = runner.runtool(args, catch)
+    out = out.strip()
+    msger.debug("exec_cmd: output for %s (rc = %d): %s" % \
+                    (cmd_and_args, rc, out))
+
+    if rc != 0:
+        # We don't throw exception when return code is not 0, because
+        # parted always fails to reload part table with loop devices. This
+        # prevents us from distinguishing real errors based on return
+        # code.
+        msger.warning("WARNING: %s returned '%s' instead of 0" % (cmd_and_args, rc))
+
+    return (rc, out)
+
+
+def exec_cmd_quiet(cmd_and_args, as_shell = False):
+    """
+    Execute command, catching nothing in the output
+
+    Need to execute as_shell if the command uses wildcards
+    """
+    return exec_cmd(cmd_and_args, as_shell, 0)
+
+
+def exec_native_cmd(cmd_and_args, native_sysroot, catch = 3):
+    """
+    Execute native command, catching stderr, stdout
+
+    Need to execute as_shell if the command uses wildcards
+
+    Always need to execute native commands as_shell
+    """
+    native_paths = \
+        "export PATH=%s/sbin:%s/usr/sbin:%s/usr/bin:$PATH" % \
+        (native_sysroot, native_sysroot, native_sysroot)
+    native_cmd_and_args = "%s;%s" % (native_paths, cmd_and_args)
+    msger.debug("exec_native_cmd: %s" % cmd_and_args)
+
+    args = cmd_and_args.split()
+    msger.debug(args)
+
+    rc, out = exec_cmd(native_cmd_and_args, True, catch)
+
+    if rc == 127: # shell command-not-found
+        msger.error("A native (host) program required to build the image "
+                    "was not found (see details above). Please make sure "
+                    "it's installed and try again.")
+
+    return (rc, out)
+
+
+def exec_native_cmd_quiet(cmd_and_args, native_sysroot):
+    """
+    Execute native command, catching nothing in the output
+
+    Need to execute as_shell if the command uses wildcards
+
+    Always need to execute native commands as_shell
+    """
+    return exec_native_cmd(cmd_and_args, native_sysroot, 0)
+
+
+# kickstart doesn't support variable substution in commands, so this
+# is our current simplistic scheme for supporting that
+
+wks_vars = dict()
+
+def get_wks_var(key):
+    return wks_vars[key]
+
+def add_wks_var(key, val):
+    wks_vars[key] = val
+
+BOOTDD_EXTRA_SPACE = 16384
+IMAGE_EXTRA_SPACE = 10240
+
+__bitbake_env_lines = ""
+
+def set_bitbake_env_lines(bitbake_env_lines):
+    global __bitbake_env_lines
+    __bitbake_env_lines = bitbake_env_lines
+
+def get_bitbake_env_lines():
+    return __bitbake_env_lines
+
+def get_line_val(line, key):
+    """
+    Extract the value from the VAR="val" string
+    """
+    if line.startswith(key + "="):
+        stripped_line = line.split('=')[1]
+        stripped_line = stripped_line.replace('\"', '')
+        return stripped_line
+    return None
+
+def get_bitbake_var(key):
+    for line in __bitbake_env_lines.split('\n'):
+        if (get_line_val(line, key)):
+            val = get_line_val(line, key)
+            return val
+    return None
diff --git a/scripts/lib/mic/utils/oe/package_manager.py b/scripts/lib/mic/utils/oe/package_manager.py
new file mode 100644
index 0000000000..92ce98e2ce
--- /dev/null
+++ b/scripts/lib/mic/utils/oe/package_manager.py
@@ -0,0 +1,810 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (c) 2014, Enea AB.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION
+# This implements the opkg package manager wrapper as a combination of
+# meta/lib/oe/package_manager.py and bitbake/lib/bb/utils.py files and
+# adaptation of those files to 'wic'.
+#
+# AUTHORS
+# Adrian Calianu <adrian.calianu (at] enea.com>
+#
+# This file incorporates work covered by the following copyright and
+# permission notice:
+#
+#     meta/COPYING.GPLv2 (GPLv2)
+#     meta/COPYING.MIT (MIT)
+#
+#     Copyright (C) 2004 Michael Lauer
+#
+#     Permission to use, copy, modify, and/or distribute this software
+#     for any purpose with or without fee is hereby granted, provided
+#     that the above copyright notice and this permission notice appear
+#     in all copies.
+#
+#     THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
+#     WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+#     WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
+#     AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR
+#     CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+#     OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+#     NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+#     CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
+from abc import ABCMeta, abstractmethod
+import os
+import glob
+import subprocess
+import shutil
+import multiprocessing
+import re
+import errno
+import fcntl
+
+from mic.utils.oe.misc import *
+from mic import msger
+
+def mkdirhier(directory):
+    """Create a directory like 'mkdir -p', but does not complain if
+    directory already exists like os.makedirs
+    """
+
+    try:
+        os.makedirs(directory)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise e
+
+def remove(path, recurse=False):
+    """Equivalent to rm -f or rm -rf"""
+    if not path:
+        return
+    if recurse:
+        # shutil.rmtree(name) would be ideal but its too slow
+        subprocess.call(['rm', '-rf'] + glob.glob(path))
+        return
+    for name in glob.glob(path):
+        try:
+            os.unlink(name)
+        except OSError as exc:
+            if exc.errno != errno.ENOENT:
+                raise
+
+def lockfile(name, shared=False, retry=True):
+    """
+    Use the file fn as a lock file, return when the lock has been acquired.
+    Returns a variable to pass to unlockfile().
+    """
+    dirname = os.path.dirname(name)
+    mkdirhier(dirname)
+
+    if not os.access(dirname, os.W_OK):
+        logger.error("Unable to acquire lock '%s', directory is not writable",
+                     name)
+        sys.exit(1)
+
+    op = fcntl.LOCK_EX
+    if shared:
+        op = fcntl.LOCK_SH
+    if not retry:
+        op = op | fcntl.LOCK_NB
+
+    while True:
+        # If we leave the lockfiles lying around there is no problem
+        # but we should clean up after ourselves. This gives potential
+        # for races though. To work around this, when we acquire the lock
+        # we check the file we locked was still the lock file on disk.
+        # by comparing inode numbers. If they don't match or the lockfile
+        # no longer exists, we start again.
+
+        # This implementation is unfair since the last person to request the
+        # lock is the most likely to win it.
+
+        try:
+            lf = open(name, 'a+')
+            fileno = lf.fileno()
+            fcntl.flock(fileno, op)
+            statinfo = os.fstat(fileno)
+            if os.path.exists(lf.name):
+                statinfo2 = os.stat(lf.name)
+                if statinfo.st_ino == statinfo2.st_ino:
+                    return lf
+            lf.close()
+        except Exception:
+            try:
+                lf.close()
+            except Exception:
+                pass
+            pass
+        if not retry:
+            return None
+
+def unlockfile(lf):
+    """
+    Unlock a file locked using lockfile()
+    """
+    try:
+        # If we had a shared lock, we need to promote to exclusive before
+        # removing the lockfile. Attempt this, ignore failures.
+        fcntl.flock(lf.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
+        os.unlink(lf.name)
+    except (IOError, OSError):
+        pass
+    fcntl.flock(lf.fileno(), fcntl.LOCK_UN)
+    lf.close()
+
+def which(path, item, direction = 0, history = False):
+    """
+    Locate a file in a PATH
+    """
+
+    hist = []
+    paths = (path or "").split(':')
+    if direction != 0:
+        paths.reverse()
+
+    for p in paths:
+        next = os.path.join(p, item)
+        hist.append(next)
+        if os.path.exists(next):
+            if not os.path.isabs(next):
+                next = os.path.abspath(next)
+            if history:
+                return next, hist
+            return next
+
+    if history:
+        return "", hist
+    return ""
+
+
+
+# this can be used by all PM backends to create the index files in parallel
+def wic_create_index(arg):
+    index_cmd = arg
+
+    try:
+        msger.info("Executing '%s' ..." % index_cmd)
+        subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True)
+    except subprocess.CalledProcessError as e:
+        return("Index creation command '%s' failed with return code %d:\n%s" %
+               (e.cmd, e.returncode, e.output))
+
+    return None
+
+
+class WicIndexer(object):
+    __metaclass__ = ABCMeta
+
+    def __init__(self, d, deploy_dir):
+        self.d = d
+        self.deploy_dir = deploy_dir
+
+    @abstractmethod
+    def write_index(self):
+        pass
+
+class WicOpkgIndexer(WicIndexer):
+    def write_index(self):
+        arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS",
+                     "SDK_PACKAGE_ARCHS",
+                     "MULTILIB_ARCHS"]
+
+        opkg_index_cmd = which(os.getenv('PATH'), "opkg-make-index")
+
+        if not os.path.exists(os.path.join(self.deploy_dir, "Packages")):
+            open(os.path.join(self.deploy_dir, "Packages"), "w").close()
+
+        index_cmds = []
+        for arch_var in arch_vars:
+            if self.d.has_key(arch_var):
+                archs = self.d[arch_var]
+            else:
+                archs = None
+
+            if archs is None:
+                continue
+
+            for arch in archs.split():
+                pkgs_dir = os.path.join(self.deploy_dir, arch)
+                pkgs_file = os.path.join(pkgs_dir, "Packages")
+
+                if not os.path.isdir(pkgs_dir):
+                    continue
+
+                if not os.path.exists(pkgs_file):
+                    open(pkgs_file, "w").close()
+
+                index_cmds.append('%s -r %s -p %s -m %s' %
+                                  (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir))
+
+        if len(index_cmds) == 0:
+            msger.info("There are no packages in %s!" % self.deploy_dir)
+            return
+
+        nproc = multiprocessing.cpu_count()
+        pool = multiprocessing.Pool(nproc)
+        results = list(pool.imap(wic_create_index, index_cmds))
+        pool.close()
+        pool.join()
+
+        for result in results:
+            if result is not None:
+                return(result)
+
+class WicPkgsList(object):
+    __metaclass__ = ABCMeta
+
+    def __init__(self, d, rootfs_dir):
+        self.d = d
+        self.rootfs_dir = rootfs_dir
+
+    @abstractmethod
+    def list(self, format=None):
+        pass
+
+
+class WicOpkgPkgsList(WicPkgsList):
+    def __init__(self, d, rootfs_dir, config_file):
+        super(WicOpkgPkgsList, self).__init__(d, rootfs_dir)
+
+        self.opkg_cmd = which(os.getenv('PATH'), "opkg-cl")
+        self.opkg_args = "-f %s -o %s " % (config_file, rootfs_dir)
+        if self.d.has_key("OPKG_ARGS"):
+            self.opkg_args += self.d["OPKG_ARGS"]
+
+    def list(self, format=None):
+        opkg_query_cmd = which(os.getenv('PATH'), "opkg-query-helper.py")
+
+        if format == "arch":
+            cmd = "%s %s status | %s -a" % \
+                (self.opkg_cmd, self.opkg_args, opkg_query_cmd)
+        elif format == "file":
+            cmd = "%s %s status | %s -f" % \
+                (self.opkg_cmd, self.opkg_args, opkg_query_cmd)
+        elif format == "ver":
+            cmd = "%s %s status | %s -v" % \
+                (self.opkg_cmd, self.opkg_args, opkg_query_cmd)
+        elif format == "deps":
+            cmd = "%s %s status | %s" % \
+                (self.opkg_cmd, self.opkg_args, opkg_query_cmd)
+        else:
+            cmd = "%s %s list_installed | cut -d' ' -f1" % \
+                (self.opkg_cmd, self.opkg_args)
+
+        try:
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip()
+        except subprocess.CalledProcessError as e:
+            msger.error("Cannot get the installed packages list. Command '%s' "
+                     "returned %d:\n%s" % (cmd, e.returncode, e.output))
+
+        if output and format == "file":
+            tmp_output = ""
+            for line in output.split('\n'):
+                pkg, pkg_file, pkg_arch = line.split()
+                full_path = os.path.join(self.rootfs_dir, pkg_arch, pkg_file)
+                if os.path.exists(full_path):
+                    tmp_output += "%s %s %s\n" % (pkg, full_path, pkg_arch)
+                else:
+                    tmp_output += "%s %s %s\n" % (pkg, pkg_file, pkg_arch)
+
+            output = tmp_output
+
+        return output
+
+
+class WicPackageManager(object):
+    """
+    This is an abstract class. Do not instantiate this directly.
+    """
+    __metaclass__ = ABCMeta
+
+    def __init__(self, d, pseudo, native_sysroot):
+        self.d = d
+        self.deploy_dir = None
+        self.deploy_lock = None
+        if self.d.has_key('PACKAGE_FEED_URIS'):
+            self.feed_uris = self.d['PACKAGE_FEED_URIS']
+        else:
+            self.feed_uris = ""
+        self.pseudo = pseudo
+        self.native_sysroot = native_sysroot
+
+    """
+    Update the package manager package database.
+    """
+    @abstractmethod
+    def update(self):
+        pass
+
+    """
+    Install a list of packages. 'pkgs' is a list object. If 'attempt_only' is
+    True, installation failures are ignored.
+    """
+    @abstractmethod
+    def install(self, pkgs, attempt_only=False):
+        pass
+
+    """
+    Remove a list of packages. 'pkgs' is a list object. If 'with_dependencies'
+    is False, the any dependencies are left in place.
+    """
+    @abstractmethod
+    def remove(self, pkgs, with_dependencies=True):
+        pass
+
+    """
+    This function creates the index files
+    """
+    @abstractmethod
+    def write_index(self):
+        pass
+
+    @abstractmethod
+    def remove_packaging_data(self):
+        pass
+
+    @abstractmethod
+    def list_installed(self, format=None):
+        pass
+
+    @abstractmethod
+    def insert_feeds_uris(self):
+        pass
+
+    """
+    Install complementary packages based upon the list of currently installed
+    packages e.g. locales, *-dev, *-dbg, etc. This will only attempt to install
+    these packages, if they don't exist then no error will occur.  Note: every
+    backend needs to call this function explicitly after the normal package
+    installation
+    """
+    def install_complementary(self, globs=None):
+        # we need to write the list of installed packages to a file because the
+        # oe-pkgdata-util reads it from a file
+        if self.d.has_key('WORKDIR'):
+            installed_pkgs_file = os.path.join(self.d['WORKDIR'],
+                                           "installed_pkgs.txt")
+        else:
+            msger.error("No WORKDIR provided!")
+
+        with open(installed_pkgs_file, "w+") as installed_pkgs:
+            installed_pkgs.write(self.list_installed("arch"))
+
+        if globs is None:
+            if self.d.has_key('IMAGE_INSTALL_COMPLEMENTARY'):
+                globs = self.d['IMAGE_INSTALL_COMPLEMENTARY']
+            split_linguas = set()
+
+            if self.d.has_key('IMAGE_LINGUAS'):
+                for translation in self.d['IMAGE_LINGUAS'].split():
+                    split_linguas.add(translation)
+                    split_linguas.add(translation.split('-')[0])
+
+            split_linguas = sorted(split_linguas)
+
+            for lang in split_linguas:
+                globs += " *-locale-%s" % lang
+
+        if globs is None:
+            return
+
+        if not self.d.has_key('PKGDATA_DIR'):
+            msger.error("No PKGDATA_DIR provided!")
+
+        cmd = [which(os.getenv('PATH'), "oe-pkgdata-util"),
+               "glob", self.d['PKGDATA_DIR'], installed_pkgs_file,
+               globs]
+
+        rc, out = exec_native_cmd(self.pseudo + cmd, self.native_sysroot)
+        if rc != 0:
+            msger.error("Could not compute complementary packages list. Command "
+                     "'%s' returned %d" %
+                     (' '.join(cmd), rc))
+
+        self.install(out.split(), attempt_only=True)
+
+
+    def deploy_dir_lock(self):
+        if self.deploy_dir is None:
+            raise RuntimeError("deploy_dir is not set!")
+
+        lock_file_name = os.path.join(self.deploy_dir, "deploy.lock")
+
+        self.deploy_lock = lockfile(lock_file_name)
+
+    def deploy_dir_unlock(self):
+        if self.deploy_lock is None:
+            return
+
+        unlockfile(self.deploy_lock)
+
+        self.deploy_lock = None
+
+
+class WicOpkgPM(WicPackageManager):
+    def __init__(self, d, target_rootfs, config_file, archs, pseudo, native_sysroot, task_name='target'):
+        super(WicOpkgPM, self).__init__(d, pseudo, native_sysroot)
+
+        self.target_rootfs = target_rootfs
+        self.config_file = config_file
+        self.pkg_archs = archs
+        self.task_name = task_name
+
+        if self.d.has_key("DEPLOY_DIR_IPK"):
+            self.deploy_dir = self.d["DEPLOY_DIR_IPK"]
+
+        self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
+        self.opkg_cmd = which(os.getenv('PATH'), "opkg-cl")
+        self.opkg_args = "-f %s -o %s " % (self.config_file, target_rootfs)
+        if self.d.has_key("OPKG_ARGS"):
+            self.opkg_args += self.d["OPKG_ARGS"]
+
+        if self.d.has_key('OPKGLIBDIR'):
+            opkg_lib_dir = self.d['OPKGLIBDIR']
+        else:
+            opkg_lib_dir = ""
+
+        if opkg_lib_dir[0] == "/":
+            opkg_lib_dir = opkg_lib_dir[1:]
+
+        self.opkg_dir = os.path.join(target_rootfs, opkg_lib_dir, "opkg")
+
+        mkdirhier(self.opkg_dir)
+
+        if self.d.has_key("TMPDIR"):
+            tmp_dir = self.d["TMPDIR"]
+        else:
+            tmp_dir = ""
+
+        self.saved_opkg_dir = '%s/saved/%s' % (tmp_dir, self.task_name)
+        if not os.path.exists('%s/saved' % tmp_dir):
+            mkdirhier('%s/saved' % tmp_dir)
+
+        if self.d.has_key('BUILD_IMAGES_FROM_FEEDS') and self.d['BUILD_IMAGES_FROM_FEEDS'] != "1":
+            self._create_config()
+        else:
+            self._create_custom_config()
+
+        self.indexer = WicOpkgIndexer(self.d, self.deploy_dir)
+
+    """
+    This function will change a package's status in /var/lib/opkg/status file.
+    If 'packages' is None then the new_status will be applied to all
+    packages
+    """
+    def mark_packages(self, status_tag, packages=None):
+        status_file = os.path.join(self.opkg_dir, "status")
+
+        with open(status_file, "r") as sf:
+            with open(status_file + ".tmp", "w+") as tmp_sf:
+                if packages is None:
+                    tmp_sf.write(re.sub(r"Package: (.*?)\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)",
+                                        r"Package: \1\n\2Status: \3%s" % status_tag,
+                                        sf.read()))
+                else:
+                    if type(packages).__name__ != "list":
+                        raise TypeError("'packages' should be a list object")
+
+                    status = sf.read()
+                    for pkg in packages:
+                        status = re.sub(r"Package: %s\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)" % pkg,
+                                        r"Package: %s\n\1Status: \2%s" % (pkg, status_tag),
+                                        status)
+
+                    tmp_sf.write(status)
+
+        os.rename(status_file + ".tmp", status_file)
+
+    def _create_custom_config(self):
+        msger.info("Building from feeds activated!")
+
+        with open(self.config_file, "w+") as config_file:
+            priority = 1
+            for arch in self.pkg_archs.split():
+                config_file.write("arch %s %d\n" % (arch, priority))
+                priority += 5
+
+            if self.d.has_key('IPK_FEED_URIS'):
+                ipk_feed_uris = self.d['IPK_FEED_URIS']
+            else:
+                ipk_feed_uris = ""
+
+            for line in ipk_feed_uris.split():
+                feed_match = re.match("^[ \t]*(.*)##([^ \t]*)[ \t]*$", line)
+
+                if feed_match is not None:
+                    feed_name = feed_match.group(1)
+                    feed_uri = feed_match.group(2)
+
+                    msger.info("Add %s feed with URL %s" % (feed_name, feed_uri))
+
+                    config_file.write("src/gz %s %s\n" % (feed_name, feed_uri))
+
+            """
+            Allow to use package deploy directory contents as quick devel-testing
+            feed. This creates individual feed configs for each arch subdir of those
+            specified as compatible for the current machine.
+            NOTE: Development-helper feature, NOT a full-fledged feed.
+            """
+            if self.d.has_key('FEED_DEPLOYDIR_BASE_URI'):
+                feed_deploydir_base_dir = self.d['FEED_DEPLOYDIR_BASE_URI']
+            else:
+                feed_deploydir_base_dir = ""
+
+            if feed_deploydir_base_dir != "":
+                for arch in self.pkg_archs.split():
+                    if self.d.has_key("sysconfdir"):
+                        sysconfdir = self.d["sysconfdir"]
+                    else:
+                        sysconfdir = None
+
+                    cfg_file_name = os.path.join(self.target_rootfs,
+                                                 sysconfdir,
+                                                 "opkg",
+                                                 "local-%s-feed.conf" % arch)
+
+                    with open(cfg_file_name, "w+") as cfg_file:
+                        cfg_file.write("src/gz local-%s %s/%s" %
+                                       arch,
+                                       feed_deploydir_base_dir,
+                                       arch)
+
+    def _create_config(self):
+        with open(self.config_file, "w+") as config_file:
+            priority = 1
+            for arch in self.pkg_archs.split():
+                config_file.write("arch %s %d\n" % (arch, priority))
+                priority += 5
+
+            config_file.write("src oe file:%s\n" % self.deploy_dir)
+
+            for arch in self.pkg_archs.split():
+                pkgs_dir = os.path.join(self.deploy_dir, arch)
+                if os.path.isdir(pkgs_dir):
+                    config_file.write("src oe-%s file:%s\n" %
+                                      (arch, pkgs_dir))
+
+    def insert_feeds_uris(self):
+        if self.feed_uris == "":
+            return
+
+        rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf'
+                                  % self.target_rootfs)
+
+        with open(rootfs_config, "w+") as config_file:
+            uri_iterator = 0
+            for uri in self.feed_uris.split():
+                config_file.write("src/gz url-%d %s/ipk\n" %
+                                  (uri_iterator, uri))
+
+                for arch in self.pkg_archs.split():
+                    if not os.path.exists(os.path.join(self.deploy_dir, arch)):
+                        continue
+                    msger.info('Note: adding opkg channel url-%s-%d (%s)' %
+                        (arch, uri_iterator, uri))
+
+                    config_file.write("src/gz uri-%s-%d %s/ipk/%s\n" %
+                                      (arch, uri_iterator, uri, arch))
+                uri_iterator += 1
+
+    def update(self):
+        self.deploy_dir_lock()
+
+        cmd = "%s %s update" % (self.opkg_cmd, self.opkg_args)
+
+        rc, out = exec_native_cmd(self.pseudo + cmd, self.native_sysroot)
+        if rc != 0:
+            self.deploy_dir_unlock()
+            msger.error("Unable to update the package index files. Command '%s' "
+                     "returned %d" % (cmd, rc))
+
+        self.deploy_dir_unlock()
+
+    def install(self, pkgs, attempt_only=False):
+        if attempt_only and len(pkgs) == 0:
+            return
+
+        cmd = "%s %s install %s" % (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
+
+        os.environ['D'] = self.target_rootfs
+        os.environ['OFFLINE_ROOT'] = self.target_rootfs
+        os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs
+        os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs
+        if self.d.has_key('WORKDIR'):
+            os.environ['INTERCEPT_DIR'] = os.path.join(self.d['WORKDIR'],
+                                                   "intercept_scripts")
+        else:
+            os.environ['INTERCEPT_DIR'] = "."
+            msger.warning("No WORKDIR provided!")
+
+        if self.d.has_key('STAGING_DIR_NATIVE'):
+            os.environ['NATIVE_ROOT'] = self.d['STAGING_DIR_NATIVE']
+        else:
+            msger.error("No STAGING_DIR_NATIVE provided!")
+
+        rc, out = exec_native_cmd(self.pseudo + cmd, self.native_sysroot)
+        if rc != 0:
+            msger.error("Unable to install packages. "
+                        "Command '%s' returned %d" % (cmd, rc))
+
+
+    def remove(self, pkgs, with_dependencies=True):
+        if with_dependencies:
+            cmd = "%s %s --force-depends --force-remove --force-removal-of-dependent-packages remove %s" % \
+                (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
+        else:
+            cmd = "%s %s --force-depends remove %s" % \
+                (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
+
+        rc, out = exec_native_cmd(self.pseudo + cmd, self.native_sysroot)
+        if rc != 0:
+            msger.error("Unable to remove packages. Command '%s' "
+                     "returned %d" % (cmd, rc))
+
+
+    def write_index(self):
+        self.deploy_dir_lock()
+
+        result = self.indexer.write_index()
+
+        self.deploy_dir_unlock()
+
+        if result is not None:
+            msger.error(result)
+
+    def remove_packaging_data(self):
+        remove(self.opkg_dir, True)
+        # create the directory back, it's needed by PM lock
+        mkdirhier(self.opkg_dir)
+
+    def list_installed(self, format=None):
+        return WicOpkgPkgsList(self.d, self.target_rootfs, self.config_file).list(format)
+
+    def handle_bad_recommendations(self):
+        if self.d.has_key("BAD_RECOMMENDATIONS"):
+            bad_recommendations = self.d["BAD_RECOMMENDATIONS"]
+        else:
+            bad_recommendations = ""
+
+        if bad_recommendations.strip() == "":
+            return
+
+        status_file = os.path.join(self.opkg_dir, "status")
+
+        # If status file existed, it means the bad recommendations has already
+        # been handled
+        if os.path.exists(status_file):
+            return
+
+        cmd = "%s %s info " % (self.opkg_cmd, self.opkg_args)
+
+        with open(status_file, "w+") as status:
+            for pkg in bad_recommendations.split():
+                pkg_info = cmd + pkg
+
+                try:
+                    output = subprocess.check_output(pkg_info.split(), stderr=subprocess.STDOUT).strip()
+                except subprocess.CalledProcessError as e:
+                    msger.error("Cannot get package info. Command '%s' "
+                             "returned %d:\n%s" % (pkg_info, e.returncode, e.output))
+
+                if output == "":
+                    msger.info("Ignored bad recommendation: '%s' is "
+                            "not a package" % pkg)
+                    continue
+
+                for line in output.split('\n'):
+                    if line.startswith("Status:"):
+                        status.write("Status: deinstall hold not-installed\n")
+                    else:
+                        status.write(line + "\n")
+
+    '''
+    The following function dummy installs pkgs and returns the log of output.
+    '''
+    def dummy_install(self, pkgs):
+        if len(pkgs) == 0:
+            return
+
+        # Create an temp dir as opkg root for dummy installation
+        if self.d.has_key("TMPDIR"):
+            tmp_dir = self.d["TMPDIR"]
+        else:
+            tmp_dir = "."
+            msger.warning("No TMPDIR provided!")
+
+        temp_rootfs = '%s/opkg' % tmp_dir
+        temp_opkg_dir = os.path.join(temp_rootfs, 'var/lib/opkg')
+        mkdirhier(temp_opkg_dir)
+
+        opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs)
+        if self.d.has_key("OPKG_ARGS"):
+            opkg_args += self.d["OPKG_ARGS"]
+
+        cmd = "%s %s update" % (self.opkg_cmd, opkg_args)
+        try:
+            subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+        except subprocess.CalledProcessError as e:
+            msger.error("Unable to update. Command '%s' "
+                     "returned %d:\n%s" % (cmd, e.returncode, e.output))
+
+        # Dummy installation
+        cmd = "%s %s --noaction install %s " % (self.opkg_cmd,
+                                                opkg_args,
+                                                ' '.join(pkgs))
+        try:
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+        except subprocess.CalledProcessError as e:
+            msger.error("Unable to dummy install packages. Command '%s' "
+                     "returned %d:\n%s" % (cmd, e.returncode, e.output))
+
+        remove(temp_rootfs, True)
+
+        return output
+
+    def backup_packaging_data(self):
+        # Save the opkglib for increment ipk image generation
+        if os.path.exists(self.saved_opkg_dir):
+            remove(self.saved_opkg_dir, True)
+        shutil.copytree(self.opkg_dir,
+                        self.saved_opkg_dir,
+                        symlinks=True)
+
+    def recover_packaging_data(self):
+        # Move the opkglib back
+        if os.path.exists(self.saved_opkg_dir):
+            if os.path.exists(self.opkg_dir):
+                remove(self.opkg_dir, True)
+
+            msger.info('Recover packaging data')
+            shutil.copytree(self.saved_opkg_dir,
+                            self.opkg_dir,
+                            symlinks=True)
+
+
+def wic_generate_index_files(d):
+    if d.has_key('PACKAGE_CLASSES'):
+        classes = d['PACKAGE_CLASSES'].replace("package_", "").split()
+    else:
+        classes = ""
+        msger.warning("No PACKAGE_CLASSES provided!")
+
+    if d.has_key('DEPLOY_DIR_IPK'):
+        deploy_dir_ipk = d['DEPLOY_DIR_IPK']
+    else:
+        deploy_dir_ipk = None
+        msger.warning("No DEPLOY_DIR_IPK provided!")
+
+    indexer_map = {
+        "ipk": (WicOpkgIndexer, deploy_dir_ipk)
+    }
+
+    result = None
+
+    for pkg_class in classes:
+        if not pkg_class in indexer_map:
+            continue
+
+        if os.path.exists(indexer_map[pkg_class][1]):
+            result = indexer_map[pkg_class][0](d, indexer_map[pkg_class][1]).write_index()
+
+            if result is not None:
+                msger.error(result)
diff --git a/scripts/lib/mic/utils/partitionedfs.py b/scripts/lib/mic/utils/partitionedfs.py
new file mode 100644
index 0000000000..6607466a83
--- /dev/null
+++ b/scripts/lib/mic/utils/partitionedfs.py
@@ -0,0 +1,782 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2009, 2010, 2011 Intel, Inc.
+# Copyright (c) 2007, 2008 Red Hat, Inc.
+# Copyright (c) 2008 Daniel P. Berrange
+# Copyright (c) 2008 David P. Huff
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os
+
+from mic import msger
+from mic.utils import runner
+from mic.utils.errors import MountError
+from mic.utils.fs_related import *
+from mic.utils.gpt_parser import GptParser
+from mic.utils.oe.misc import *
+
+# Overhead of the MBR partitioning scheme (just one sector)
+MBR_OVERHEAD = 1
+# Overhead of the GPT partitioning scheme
+GPT_OVERHEAD = 34
+
+# Size of a sector in bytes
+SECTOR_SIZE = 512
+
+class PartitionedMount(Mount):
+    def __init__(self, mountdir, skipformat = False):
+        Mount.__init__(self, mountdir)
+        self.disks = {}
+        self.partitions = []
+        self.subvolumes = []
+        self.mapped = False
+        self.mountOrder = []
+        self.unmountOrder = []
+        self.parted = find_binary_path("parted")
+        self.btrfscmd=None
+        self.skipformat = skipformat
+        self.snapshot_created = self.skipformat
+        # Size of a sector used in calculations
+        self.sector_size = SECTOR_SIZE
+        self._partitions_layed_out = False
+
+    def __add_disk(self, disk_name):
+        """ Add a disk 'disk_name' to the internal list of disks. Note,
+        'disk_name' is the name of the disk in the target system
+        (e.g., sdb). """
+
+        if disk_name in self.disks:
+            # We already have this disk
+            return
+
+        assert not self._partitions_layed_out
+
+        self.disks[disk_name] = \
+                { 'disk': None,     # Disk object
+                  'mapped': False,  # True if kpartx mapping exists
+                  'numpart': 0,     # Number of allocate partitions
+                  'partitions': [], # Indexes to self.partitions
+                  'offset': 0,      # Offset of next partition (in sectors)
+                  # Minimum required disk size to fit all partitions (in bytes)
+                  'min_size': 0,
+                  'ptable_format': "msdos" } # Partition table format
+
+    def add_disk(self, disk_name, disk_obj):
+        """ Add a disk object which have to be partitioned. More than one disk
+        can be added. In case of multiple disks, disk partitions have to be
+        added for each disk separately with 'add_partition()". """
+
+        self.__add_disk(disk_name)
+        self.disks[disk_name]['disk'] = disk_obj
+
+    def __add_partition(self, part):
+        """ This is a helper function for 'add_partition()' which adds a
+        partition to the internal list of partitions. """
+
+        assert not self._partitions_layed_out
+
+        self.partitions.append(part)
+        self.__add_disk(part['disk_name'])
+
+    def add_partition(self, size, disk_name, mountpoint, source_file = None, fstype = None,
+                      label=None, fsopts = None, boot = False, align = None,
+                      part_type = None):
+        """ Add the next partition. Prtitions have to be added in the
+        first-to-last order. """
+
+        ks_pnum = len(self.partitions)
+
+        # Converting MB to sectors for parted
+        size = size * 1024 * 1024 / self.sector_size
+
+        # We need to handle subvolumes for btrfs
+        if fstype == "btrfs" and fsopts and fsopts.find("subvol=") != -1:
+            self.btrfscmd=find_binary_path("btrfs")
+            subvol = None
+            opts = fsopts.split(",")
+            for opt in opts:
+                if opt.find("subvol=") != -1:
+                    subvol = opt.replace("subvol=", "").strip()
+                    break
+            if not subvol:
+                raise MountError("No subvolume: %s" % fsopts)
+            self.subvolumes.append({'size': size, # In sectors
+                                    'mountpoint': mountpoint, # Mount relative to chroot
+                                    'fstype': fstype, # Filesystem type
+                                    'fsopts': fsopts, # Filesystem mount options
+                                    'disk_name': disk_name, # physical disk name holding partition
+                                    'device': None, # kpartx device node for partition
+                                    'mount': None, # Mount object
+                                    'subvol': subvol, # Subvolume name
+                                    'boot': boot, # Bootable flag
+                                    'mounted': False # Mount flag
+                                   })
+
+        # We still need partition for "/" or non-subvolume
+        if mountpoint == "/" or not fsopts or fsopts.find("subvol=") == -1:
+            # Don't need subvolume for "/" because it will be set as default subvolume
+            if fsopts and fsopts.find("subvol=") != -1:
+                opts = fsopts.split(",")
+                for opt in opts:
+                    if opt.strip().startswith("subvol="):
+                        opts.remove(opt)
+                        break
+                fsopts = ",".join(opts)
+
+            part = { 'ks_pnum' : ks_pnum, # Partition number in the KS file
+                     'size': size, # In sectors
+                     'mountpoint': mountpoint, # Mount relative to chroot
+                     'source_file': source_file, # partition contents
+                     'fstype': fstype, # Filesystem type
+                     'fsopts': fsopts, # Filesystem mount options
+                     'label': label, # Partition label
+                     'disk_name': disk_name, # physical disk name holding partition
+                     'device': None, # kpartx device node for partition
+                     'mount': None, # Mount object
+                     'num': None, # Partition number
+                     'boot': boot, # Bootable flag
+                     'align': align, # Partition alignment
+                     'part_type' : part_type, # Partition type
+                     'partuuid': None } # Partition UUID (GPT-only)
+
+            self.__add_partition(part)
+
+    def layout_partitions(self, ptable_format = "msdos"):
+        """ Layout the partitions, meaning calculate the position of every
+        partition on the disk. The 'ptable_format' parameter defines the
+        partition table format, and may be either "msdos" or "gpt". """
+
+        msger.debug("Assigning %s partitions to disks" % ptable_format)
+
+        if ptable_format not in ('msdos', 'gpt'):
+            raise MountError("Unknown partition table format '%s', supported " \
+                             "formats are: 'msdos' and 'gpt'" % ptable_format)
+
+        if self._partitions_layed_out:
+            return
+
+        self._partitions_layed_out = True
+
+        # Go through partitions in the order they are added in .ks file
+        for n in range(len(self.partitions)):
+            p = self.partitions[n]
+
+            if not self.disks.has_key(p['disk_name']):
+                raise MountError("No disk %s for partition %s" \
+                                 % (p['disk_name'], p['mountpoint']))
+
+            if p['part_type'] and ptable_format != 'gpt':
+                # The --part-type can also be implemented for MBR partitions,
+                # in which case it would map to the 1-byte "partition type"
+                # filed at offset 3 of the partition entry.
+                raise MountError("setting custom partition type is only " \
+                                 "imlemented for GPT partitions")
+
+            # Get the disk where the partition is located
+            d = self.disks[p['disk_name']]
+            d['numpart'] += 1
+            d['ptable_format'] = ptable_format
+
+            if d['numpart'] == 1:
+                if ptable_format == "msdos":
+                    overhead = MBR_OVERHEAD
+                else:
+                    overhead = GPT_OVERHEAD
+
+                # Skip one sector required for the partitioning scheme overhead
+                d['offset'] += overhead
+                # Steal few sectors from the first partition to offset for the
+                # partitioning overhead
+                p['size'] -= overhead
+
+            if p['align']:
+                # If not first partition and we do have alignment set we need
+                # to align the partition.
+                # FIXME: This leaves a empty spaces to the disk. To fill the
+                # gaps we could enlargea the previous partition?
+
+                # Calc how much the alignment is off.
+                align_sectors = d['offset'] % (p['align'] * 1024 / self.sector_size)
+                # We need to move forward to the next alignment point
+                align_sectors = (p['align'] * 1024 / self.sector_size) - align_sectors
+
+                msger.debug("Realignment for %s%s with %s sectors, original"
+                            " offset %s, target alignment is %sK." %
+                            (p['disk_name'], d['numpart'], align_sectors,
+                             d['offset'], p['align']))
+
+                # increase the offset so we actually start the partition on right alignment
+                d['offset'] += align_sectors
+
+            p['start'] = d['offset']
+            d['offset'] += p['size']
+
+            p['type'] = 'primary'
+            p['num'] = d['numpart']
+
+            if d['ptable_format'] == "msdos":
+                if d['numpart'] > 2:
+                    # Every logical partition requires an additional sector for
+                    # the EBR, so steal the last sector from the end of each
+                    # partition starting from the 3rd one for the EBR. This
+                    # will make sure the logical partitions are aligned
+                    # correctly.
+                    p['size'] -= 1
+
+                if d['numpart'] > 3:
+                    p['type'] = 'logical'
+                    p['num'] = d['numpart'] + 1
+
+            d['partitions'].append(n)
+            msger.debug("Assigned %s to %s%d, sectors range %d-%d size %d "
+                        "sectors (%d bytes)." \
+                            % (p['mountpoint'], p['disk_name'], p['num'],
+                               p['start'], p['start'] + p['size'] - 1,
+                               p['size'], p['size'] * self.sector_size))
+
+        # Once all the partitions have been layed out, we can calculate the
+        # minumim disk sizes.
+        for disk_name, d in self.disks.items():
+            d['min_size'] = d['offset']
+            if d['ptable_format'] == 'gpt':
+                # Account for the backup partition table at the end of the disk
+                d['min_size'] += GPT_OVERHEAD
+
+            d['min_size'] *= self.sector_size
+
+    def __run_parted(self, args):
+        """ Run parted with arguments specified in the 'args' list. """
+
+        args.insert(0, self.parted)
+        msger.debug(args)
+
+        rc, out = runner.runtool(args, catch = 3)
+        out = out.strip()
+        if out:
+            msger.debug('"parted" output: %s' % out)
+
+        if rc != 0:
+            # We don't throw exception when return code is not 0, because
+            # parted always fails to reload part table with loop devices. This
+            # prevents us from distinguishing real errors based on return
+            # code.
+            msger.debug("WARNING: parted returned '%s' instead of 0" % rc)
+
+    def __create_partition(self, device, parttype, fstype, start, size):
+        """ Create a partition on an image described by the 'device' object. """
+
+        # Start is included to the size so we need to substract one from the end.
+        end = start + size - 1
+        msger.debug("Added '%s' partition, sectors %d-%d, size %d sectors" %
+                    (parttype, start, end, size))
+
+        args = ["-s", device, "unit", "s", "mkpart", parttype]
+        if fstype:
+            args.extend([fstype])
+        args.extend(["%d" % start, "%d" % end])
+
+        return self.__run_parted(args)
+
+    def __format_disks(self):
+        self.layout_partitions()
+
+        if self.skipformat:
+            msger.debug("Skipping disk format, because skipformat flag is set.")
+            return
+
+        for dev in self.disks.keys():
+            d = self.disks[dev]
+            msger.debug("Initializing partition table for %s" % \
+                        (d['disk'].device))
+            self.__run_parted(["-s", d['disk'].device, "mklabel",
+                               d['ptable_format']])
+
+        msger.debug("Creating partitions")
+
+        for p in self.partitions:
+            d = self.disks[p['disk_name']]
+            if d['ptable_format'] == "msdos" and p['num'] == 5:
+                # The last sector of the 3rd partition was reserved for the EBR
+                # of the first _logical_ partition. This is why the extended
+                # partition should start one sector before the first logical
+                # partition.
+                self.__create_partition(d['disk'].device, "extended",
+                                        None, p['start'] - 1,
+                                        d['offset'] - p['start'])
+
+            if p['fstype'] == "swap":
+                parted_fs_type = "linux-swap"
+            elif p['fstype'] == "vfat":
+                parted_fs_type = "fat32"
+            elif p['fstype'] == "msdos":
+                parted_fs_type = "fat16"
+            else:
+                # Type for ext2/ext3/ext4/btrfs
+                parted_fs_type = "ext2"
+
+            # Boot ROM of OMAP boards require vfat boot partition to have an
+            # even number of sectors.
+            if p['mountpoint'] == "/boot" and p['fstype'] in ["vfat", "msdos"] \
+               and p['size'] % 2:
+                msger.debug("Substracting one sector from '%s' partition to " \
+                            "get even number of sectors for the partition" % \
+                            p['mountpoint'])
+                p['size'] -= 1
+
+            self.__create_partition(d['disk'].device, p['type'],
+                                    parted_fs_type, p['start'], p['size'])
+
+            if p['boot']:
+                if d['ptable_format'] == 'gpt':
+                    flag_name = "legacy_boot"
+                else:
+                    flag_name = "boot"
+                msger.debug("Set '%s' flag for partition '%s' on disk '%s'" % \
+                            (flag_name, p['num'], d['disk'].device))
+                self.__run_parted(["-s", d['disk'].device, "set",
+                                   "%d" % p['num'], flag_name, "on"])
+
+            # Parted defaults to enabling the lba flag for fat16 partitions,
+            # which causes compatibility issues with some firmware (and really
+            # isn't necessary).
+            if parted_fs_type == "fat16":
+                if d['ptable_format'] == 'msdos':
+                    msger.debug("Disable 'lba' flag for partition '%s' on disk '%s'" % \
+                                (p['num'], d['disk'].device))
+                    self.__run_parted(["-s", d['disk'].device, "set",
+                                       "%d" % p['num'], "lba", "off"])
+
+        # If the partition table format is "gpt", find out PARTUUIDs for all
+        # the partitions. And if users specified custom parition type UUIDs,
+        # set them.
+        for disk_name, disk in self.disks.items():
+            if disk['ptable_format'] != 'gpt':
+                continue
+
+            pnum = 0
+            gpt_parser = GptParser(d['disk'].device, SECTOR_SIZE)
+            # Iterate over all GPT partitions on this disk
+            for entry in gpt_parser.get_partitions():
+                pnum += 1
+                # Find the matching partition in the 'self.partitions' list
+                for n in d['partitions']:
+                    p = self.partitions[n]
+                    if p['num'] == pnum:
+                        # Found, fetch PARTUUID (partition's unique ID)
+                        p['partuuid'] = entry['part_uuid']
+                        msger.debug("PARTUUID for partition %d on disk '%s' " \
+                                    "(mount point '%s') is '%s'" % (pnum, \
+                                    disk_name, p['mountpoint'], p['partuuid']))
+                        if p['part_type']:
+                            entry['type_uuid'] = p['part_type']
+                            msger.debug("Change type of partition %d on disk " \
+                                        "'%s' (mount point '%s') to '%s'" % \
+                                        (pnum, disk_name, p['mountpoint'],
+                                         p['part_type']))
+                            gpt_parser.change_partition(entry)
+
+            del gpt_parser
+
+    def __map_partitions(self):
+        """Load it if dm_snapshot isn't loaded. """
+        load_module("dm_snapshot")
+
+        for dev in self.disks.keys():
+            d = self.disks[dev]
+            if d['mapped']:
+                continue
+
+            msger.debug("Running kpartx on %s" % d['disk'].device )
+            rc, kpartxOutput = runner.runtool([self.kpartx, "-l", "-v", d['disk'].device])
+            kpartxOutput = kpartxOutput.splitlines()
+
+            if rc != 0:
+                raise MountError("Failed to query partition mapping for '%s'" %
+                                 d['disk'].device)
+
+            # Strip trailing blank and mask verbose output
+            i = 0
+            while i < len(kpartxOutput) and kpartxOutput[i][0:4] != "loop":
+                i = i + 1
+            kpartxOutput = kpartxOutput[i:]
+
+            # Make sure kpartx reported the right count of partitions
+            if len(kpartxOutput) != d['numpart']:
+                # If this disk has more than 3 partitions, then in case of MBR
+                # paritions there is an extended parition. Different versions
+                # of kpartx behave differently WRT the extended partition -
+                # some map it, some ignore it. This is why we do the below hack
+                # - if kpartx reported one more partition and the partition
+                # table type is "msdos" and the amount of partitions is more
+                # than 3, we just assume kpartx mapped the extended parition
+                # and we remove it.
+                if len(kpartxOutput) == d['numpart'] + 1 \
+                   and d['ptable_format'] == 'msdos' and len(kpartxOutput) > 3:
+                    kpartxOutput.pop(3)
+                else:
+                    raise MountError("Unexpected number of partitions from " \
+                                     "kpartx: %d != %d" % \
+                                        (len(kpartxOutput), d['numpart']))
+
+            for i in range(len(kpartxOutput)):
+                line = kpartxOutput[i]
+                newdev = line.split()[0]
+                mapperdev = "/dev/mapper/" + newdev
+                loopdev = d['disk'].device + newdev[-1]
+
+                msger.debug("Dev %s: %s -> %s" % (newdev, loopdev, mapperdev))
+                pnum = d['partitions'][i]
+                self.partitions[pnum]['device'] = loopdev
+
+                # grub's install wants partitions to be named
+                # to match their parent device + partition num
+                # kpartx doesn't work like this, so we add compat
+                # symlinks to point to /dev/mapper
+                if os.path.lexists(loopdev):
+                    os.unlink(loopdev)
+                os.symlink(mapperdev, loopdev)
+
+            msger.debug("Adding partx mapping for %s" % d['disk'].device)
+            rc = runner.show([self.kpartx, "-v", "-a", d['disk'].device])
+
+            if rc != 0:
+                # Make sure that the device maps are also removed on error case.
+                # The d['mapped'] isn't set to True if the kpartx fails so
+                # failed mapping will not be cleaned on cleanup either.
+                runner.quiet([self.kpartx, "-d", d['disk'].device])
+                raise MountError("Failed to map partitions for '%s'" %
+                                 d['disk'].device)
+
+            # FIXME: there is a bit delay for multipath device setup,
+            # wait 10ms for the setup
+            import time
+            time.sleep(10)
+            d['mapped'] = True
+
+    def __unmap_partitions(self):
+        for dev in self.disks.keys():
+            d = self.disks[dev]
+            if not d['mapped']:
+                continue
+
+            msger.debug("Removing compat symlinks")
+            for pnum in d['partitions']:
+                if self.partitions[pnum]['device'] != None:
+                    os.unlink(self.partitions[pnum]['device'])
+                    self.partitions[pnum]['device'] = None
+
+            msger.debug("Unmapping %s" % d['disk'].device)
+            rc = runner.quiet([self.kpartx, "-d", d['disk'].device])
+            if rc != 0:
+                raise MountError("Failed to unmap partitions for '%s'" %
+                                 d['disk'].device)
+
+            d['mapped'] = False
+
+    def __calculate_mountorder(self):
+        msger.debug("Calculating mount order")
+        for p in self.partitions:
+            if p['mountpoint']:
+                self.mountOrder.append(p['mountpoint'])
+                self.unmountOrder.append(p['mountpoint'])
+
+        self.mountOrder.sort()
+        self.unmountOrder.sort()
+        self.unmountOrder.reverse()
+
+    def cleanup(self):
+        Mount.cleanup(self)
+        if self.disks:
+            self.__unmap_partitions()
+            for dev in self.disks.keys():
+                d = self.disks[dev]
+                try:
+                    d['disk'].cleanup()
+                except:
+                    pass
+
+    def unmount(self):
+        self.__unmount_subvolumes()
+        for mp in self.unmountOrder:
+            if mp == 'swap':
+                continue
+            p = None
+            for p1 in self.partitions:
+                if p1['mountpoint'] == mp:
+                    p = p1
+                    break
+
+            if p['mount'] != None:
+                try:
+                    # Create subvolume snapshot here
+                    if p['fstype'] == "btrfs" and p['mountpoint'] == "/" and not self.snapshot_created:
+                        self.__create_subvolume_snapshots(p, p["mount"])
+                    p['mount'].cleanup()
+                except:
+                    pass
+                p['mount'] = None
+
+    # Only for btrfs
+    def __get_subvolume_id(self, rootpath, subvol):
+        if not self.btrfscmd:
+            self.btrfscmd=find_binary_path("btrfs")
+        argv = [ self.btrfscmd, "subvolume", "list", rootpath ]
+
+        rc, out = runner.runtool(argv)
+        msger.debug(out)
+
+        if rc != 0:
+            raise MountError("Failed to get subvolume id from %s', return code: %d." % (rootpath, rc))
+
+        subvolid = -1
+        for line in out.splitlines():
+            if line.endswith(" path %s" % subvol):
+                subvolid = line.split()[1]
+                if not subvolid.isdigit():
+                    raise MountError("Invalid subvolume id: %s" % subvolid)
+                subvolid = int(subvolid)
+                break
+        return subvolid
+
+    def __create_subvolume_metadata(self, p, pdisk):
+        if len(self.subvolumes) == 0:
+            return
+
+        argv = [ self.btrfscmd, "subvolume", "list", pdisk.mountdir ]
+        rc, out = runner.runtool(argv)
+        msger.debug(out)
+
+        if rc != 0:
+            raise MountError("Failed to get subvolume id from %s', return code: %d." % (pdisk.mountdir, rc))
+
+        subvolid_items = out.splitlines()
+        subvolume_metadata = ""
+        for subvol in self.subvolumes:
+            for line in subvolid_items:
+                if line.endswith(" path %s" % subvol["subvol"]):
+                    subvolid = line.split()[1]
+                    if not subvolid.isdigit():
+                        raise MountError("Invalid subvolume id: %s" % subvolid)
+
+                    subvolid = int(subvolid)
+                    opts = subvol["fsopts"].split(",")
+                    for opt in opts:
+                        if opt.strip().startswith("subvol="):
+                            opts.remove(opt)
+                            break
+                    fsopts = ",".join(opts)
+                    subvolume_metadata += "%d\t%s\t%s\t%s\n" % (subvolid, subvol["subvol"], subvol['mountpoint'], fsopts)
+
+        if subvolume_metadata:
+            fd = open("%s/.subvolume_metadata" % pdisk.mountdir, "w")
+            fd.write(subvolume_metadata)
+            fd.close()
+
+    def __get_subvolume_metadata(self, p, pdisk):
+        subvolume_metadata_file = "%s/.subvolume_metadata" % pdisk.mountdir
+        if not os.path.exists(subvolume_metadata_file):
+            return
+
+        fd = open(subvolume_metadata_file, "r")
+        content = fd.read()
+        fd.close()
+
+        for line in content.splitlines():
+            items = line.split("\t")
+            if items and len(items) == 4:
+                self.subvolumes.append({'size': 0, # In sectors
+                                        'mountpoint': items[2], # Mount relative to chroot
+                                        'fstype': "btrfs", # Filesystem type
+                                        'fsopts': items[3] + ",subvol=%s" %  items[1], # Filesystem mount options
+                                        'disk_name': p['disk_name'], # physical disk name holding partition
+                                        'device': None, # kpartx device node for partition
+                                        'mount': None, # Mount object
+                                        'subvol': items[1], # Subvolume name
+                                        'boot': False, # Bootable flag
+                                        'mounted': False # Mount flag
+                                   })
+
+    def __create_subvolumes(self, p, pdisk):
+        """ Create all the subvolumes. """
+
+        for subvol in self.subvolumes:
+            argv = [ self.btrfscmd, "subvolume", "create", pdisk.mountdir + "/" + subvol["subvol"]]
+
+            rc = runner.show(argv)
+            if rc != 0:
+                raise MountError("Failed to create subvolume '%s', return code: %d." % (subvol["subvol"], rc))
+
+        # Set default subvolume, subvolume for "/" is default
+        subvol = None
+        for subvolume in self.subvolumes:
+            if subvolume["mountpoint"] == "/" and p['disk_name'] == subvolume['disk_name']:
+                subvol = subvolume
+                break
+
+        if subvol:
+            # Get default subvolume id
+            subvolid = self. __get_subvolume_id(pdisk.mountdir, subvol["subvol"])
+            # Set default subvolume
+            if subvolid != -1:
+                rc = runner.show([ self.btrfscmd, "subvolume", "set-default", "%d" % subvolid, pdisk.mountdir])
+                if rc != 0:
+                    raise MountError("Failed to set default subvolume id: %d', return code: %d." % (subvolid, rc))
+
+        self.__create_subvolume_metadata(p, pdisk)
+
+    def __mount_subvolumes(self, p, pdisk):
+        if self.skipformat:
+            # Get subvolume info
+            self.__get_subvolume_metadata(p, pdisk)
+            # Set default mount options
+            if len(self.subvolumes) != 0:
+                for subvol in self.subvolumes:
+                    if subvol["mountpoint"] == p["mountpoint"] == "/":
+                        opts = subvol["fsopts"].split(",")
+                        for opt in opts:
+                            if opt.strip().startswith("subvol="):
+                                opts.remove(opt)
+                                break
+                        pdisk.fsopts = ",".join(opts)
+                        break
+
+        if len(self.subvolumes) == 0:
+            # Return directly if no subvolumes
+            return
+
+        # Remount to make default subvolume mounted
+        rc = runner.show([self.umountcmd, pdisk.mountdir])
+        if rc != 0:
+            raise MountError("Failed to umount %s" % pdisk.mountdir)
+
+        rc = runner.show([self.mountcmd, "-o", pdisk.fsopts, pdisk.disk.device, pdisk.mountdir])
+        if rc != 0:
+            raise MountError("Failed to umount %s" % pdisk.mountdir)
+
+        for subvol in self.subvolumes:
+            if subvol["mountpoint"] == "/":
+                continue
+            subvolid = self. __get_subvolume_id(pdisk.mountdir, subvol["subvol"])
+            if subvolid == -1:
+                msger.debug("WARNING: invalid subvolume %s" % subvol["subvol"])
+                continue
+            # Replace subvolume name with subvolume ID
+            opts = subvol["fsopts"].split(",")
+            for opt in opts:
+                if opt.strip().startswith("subvol="):
+                    opts.remove(opt)
+                    break
+
+            opts.extend(["subvolrootid=0", "subvol=%s" % subvol["subvol"]])
+            fsopts = ",".join(opts)
+            subvol['fsopts'] = fsopts
+            mountpoint = self.mountdir + subvol['mountpoint']
+            makedirs(mountpoint)
+            rc = runner.show([self.mountcmd, "-o", fsopts, pdisk.disk.device, mountpoint])
+            if rc != 0:
+                raise MountError("Failed to mount subvolume %s to %s" % (subvol["subvol"], mountpoint))
+            subvol["mounted"] = True
+
+    def __unmount_subvolumes(self):
+        """ It may be called multiple times, so we need to chekc if it is still mounted. """
+        for subvol in self.subvolumes:
+            if subvol["mountpoint"] == "/":
+                continue
+            if not subvol["mounted"]:
+                continue
+            mountpoint = self.mountdir + subvol['mountpoint']
+            rc = runner.show([self.umountcmd, mountpoint])
+            if rc != 0:
+                raise MountError("Failed to unmount subvolume %s from %s" % (subvol["subvol"], mountpoint))
+            subvol["mounted"] = False
+
+    def __create_subvolume_snapshots(self, p, pdisk):
+        import time
+
+        if self.snapshot_created:
+            return
+
+        # Remount with subvolid=0
+        rc = runner.show([self.umountcmd, pdisk.mountdir])
+        if rc != 0:
+            raise MountError("Failed to umount %s" % pdisk.mountdir)
+        if pdisk.fsopts:
+            mountopts = pdisk.fsopts + ",subvolid=0"
+        else:
+            mountopts = "subvolid=0"
+        rc = runner.show([self.mountcmd, "-o", mountopts, pdisk.disk.device, pdisk.mountdir])
+        if rc != 0:
+            raise MountError("Failed to umount %s" % pdisk.mountdir)
+
+        # Create all the subvolume snapshots
+        snapshotts = time.strftime("%Y%m%d-%H%M")
+        for subvol in self.subvolumes:
+            subvolpath = pdisk.mountdir + "/" + subvol["subvol"]
+            snapshotpath = subvolpath + "_%s-1" % snapshotts
+            rc = runner.show([ self.btrfscmd, "subvolume", "snapshot", subvolpath, snapshotpath ])
+            if rc != 0:
+                raise MountError("Failed to create subvolume snapshot '%s' for '%s', return code: %d." % (snapshotpath, subvolpath, rc))
+
+        self.snapshot_created = True
+
+    def __install_partition(self, num, source_file, start, size):
+        """
+        Install source_file contents into a partition.
+        """
+        if not source_file: # nothing to install
+            return
+
+        # Start is included in the size so need to substract one from the end.
+        end = start + size - 1
+        msger.debug("Installed %s in partition %d, sectors %d-%d, size %d sectors" % (source_file, num, start, end, size))
+
+        dd_cmd = "dd if=%s of=%s bs=%d seek=%d count=%d conv=notrunc" % \
+            (source_file, self.image_file, self.sector_size, start, size)
+        rc, out = exec_cmd(dd_cmd)
+
+
+    def install(self, image_file):
+        msger.debug("Installing partitions")
+
+        self.image_file = image_file
+
+        for p in self.partitions:
+            d = self.disks[p['disk_name']]
+            if d['ptable_format'] == "msdos" and p['num'] == 5:
+                # The last sector of the 3rd partition was reserved for the EBR
+                # of the first _logical_ partition. This is why the extended
+                # partition should start one sector before the first logical
+                # partition.
+                self.__install_partition(p['num'], p['source_file'],
+                                         p['start'] - 1,
+                                         d['offset'] - p['start'])
+
+            self.__install_partition(p['num'], p['source_file'],
+                                     p['start'], p['size'])
+
+    def mount(self):
+        for dev in self.disks.keys():
+            d = self.disks[dev]
+            d['disk'].create()
+
+        self.__format_disks()
+
+        self.__calculate_mountorder()
+
+        return
+
+    def resparse(self, size = None):
+        # Can't re-sparse a disk image - too hard
+        pass
diff --git a/scripts/lib/mic/utils/proxy.py b/scripts/lib/mic/utils/proxy.py
new file mode 100644
index 0000000000..91451a2d01
--- /dev/null
+++ b/scripts/lib/mic/utils/proxy.py
@@ -0,0 +1,183 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2010, 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os
+import urlparse
+
+_my_proxies = {}
+_my_noproxy = None
+_my_noproxy_list = []
+
+def set_proxy_environ():
+    global _my_noproxy, _my_proxies
+    if not _my_proxies:
+        return
+    for key in _my_proxies.keys():
+        os.environ[key + "_proxy"] = _my_proxies[key]
+    if not _my_noproxy:
+        return
+    os.environ["no_proxy"] = _my_noproxy
+
+def unset_proxy_environ():
+    for env in ('http_proxy',
+                'https_proxy',
+                'ftp_proxy',
+                'all_proxy'):
+        if env in os.environ:
+            del os.environ[env]
+
+        ENV=env.upper()
+        if ENV in os.environ:
+            del os.environ[ENV]
+
+def _set_proxies(proxy = None, no_proxy = None):
+    """Return a dictionary of scheme -> proxy server URL mappings.
+    """
+
+    global _my_noproxy, _my_proxies
+    _my_proxies = {}
+    _my_noproxy = None
+    proxies = []
+    if proxy:
+        proxies.append(("http_proxy", proxy))
+    if no_proxy:
+        proxies.append(("no_proxy", no_proxy))
+
+    # Get proxy settings from environment if not provided
+    if not proxy and not no_proxy:
+        proxies = os.environ.items()
+
+        # Remove proxy env variables, urllib2 can't handle them correctly
+        unset_proxy_environ()
+
+    for name, value in proxies:
+        name = name.lower()
+        if value and name[-6:] == '_proxy':
+            if name[0:2] != "no":
+                _my_proxies[name[:-6]] = value
+            else:
+                _my_noproxy = value
+
+def _ip_to_int(ip):
+    ipint=0
+    shift=24
+    for dec in ip.split("."):
+        ipint |= int(dec) << shift
+        shift -= 8
+    return ipint
+
+def _int_to_ip(val):
+    ipaddr=""
+    shift=0
+    for i in range(4):
+        dec = val >> shift
+        dec &= 0xff
+        ipaddr = ".%d%s" % (dec, ipaddr)
+        shift += 8
+    return ipaddr[1:]
+
+def _isip(host):
+    if host.replace(".", "").isdigit():
+        return True
+    return False
+
+def _set_noproxy_list():
+    global _my_noproxy, _my_noproxy_list
+    _my_noproxy_list = []
+    if not _my_noproxy:
+        return
+    for item in _my_noproxy.split(","):
+        item = item.strip()
+        if not item:
+            continue
+
+        if item[0] != '.' and item.find("/") == -1:
+            # Need to match it
+            _my_noproxy_list.append({"match":0,"needle":item})
+
+        elif item[0] == '.':
+            # Need to match at tail
+            _my_noproxy_list.append({"match":1,"needle":item})
+
+        elif item.find("/") > 3:
+            # IP/MASK, need to match at head
+            needle = item[0:item.find("/")].strip()
+            ip = _ip_to_int(needle)
+            netmask = 0
+            mask = item[item.find("/")+1:].strip()
+
+            if mask.isdigit():
+                netmask = int(mask)
+                netmask = ~((1<<(32-netmask)) - 1)
+                ip &= netmask
+            else:
+                shift=24
+                netmask=0
+                for dec in mask.split("."):
+                    netmask |= int(dec) << shift
+                    shift -= 8
+                ip &= netmask
+
+            _my_noproxy_list.append({"match":2,"needle":ip,"netmask":netmask})
+
+def _isnoproxy(url):
+    (scheme, host, path, parm, query, frag) = urlparse.urlparse(url)
+
+    if '@' in host:
+        user_pass, host = host.split('@', 1)
+
+    if ':' in host:
+        host, port = host.split(':', 1)
+
+    hostisip = _isip(host)
+    for item in _my_noproxy_list:
+        if hostisip and item["match"] <= 1:
+            continue
+
+        if item["match"] == 2 and hostisip:
+            if (_ip_to_int(host) & item["netmask"]) == item["needle"]:
+                return True
+
+        if item["match"] == 0:
+            if host == item["needle"]:
+                return True
+
+        if item["match"] == 1:
+            if host.rfind(item["needle"]) > 0:
+                return True
+
+    return False
+
+def set_proxies(proxy = None, no_proxy = None):
+    _set_proxies(proxy, no_proxy)
+    _set_noproxy_list()
+    set_proxy_environ()
+
+def get_proxy_for(url):
+    if url.startswith('file:') or _isnoproxy(url):
+        return None
+
+    type = url[0:url.index(":")]
+    proxy = None
+    if _my_proxies.has_key(type):
+        proxy = _my_proxies[type]
+    elif _my_proxies.has_key("http"):
+        proxy = _my_proxies["http"]
+    else:
+        proxy = None
+
+    return proxy
diff --git a/scripts/lib/mic/utils/rpmmisc.py b/scripts/lib/mic/utils/rpmmisc.py
new file mode 100644
index 0000000000..af15763e18
--- /dev/null
+++ b/scripts/lib/mic/utils/rpmmisc.py
@@ -0,0 +1,600 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2008, 2009, 2010, 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os
+import sys
+import re
+import rpm
+
+from mic import msger
+from mic.utils.errors import CreatorError
+from mic.utils.proxy import get_proxy_for
+from mic.utils import runner
+
+
+class RPMInstallCallback:
+    """ Command line callback class for callbacks from the RPM library.
+    """
+
+    def __init__(self, ts, output=1):
+        self.output = output
+        self.callbackfilehandles = {}
+        self.total_actions = 0
+        self.total_installed = 0
+        self.installed_pkg_names = []
+        self.total_removed = 0
+        self.mark = "+"
+        self.marks = 40
+        self.lastmsg = None
+        self.tsInfo = None # this needs to be set for anything else to work
+        self.ts = ts
+        self.filelog = False
+        self.logString = []
+        self.headmsg = "Installing"
+
+    def _dopkgtup(self, hdr):
+        tmpepoch = hdr['epoch']
+        if tmpepoch is None: epoch = '0'
+        else: epoch = str(tmpepoch)
+
+        return (hdr['name'], hdr['arch'], epoch, hdr['version'], hdr['release'])
+
+    def _makeHandle(self, hdr):
+        handle = '%s:%s.%s-%s-%s' % (hdr['epoch'], hdr['name'], hdr['version'],
+          hdr['release'], hdr['arch'])
+
+        return handle
+
+    def _localprint(self, msg):
+        if self.output:
+            msger.info(msg)
+
+    def _makefmt(self, percent, progress = True):
+        l = len(str(self.total_actions))
+        size = "%s.%s" % (l, l)
+        fmt_done = "[%" + size + "s/%" + size + "s]"
+        done = fmt_done % (self.total_installed + self.total_removed,
+                           self.total_actions)
+        marks = self.marks - (2 * l)
+        width = "%s.%s" % (marks, marks)
+        fmt_bar = "%-" + width + "s"
+        if progress:
+            bar = fmt_bar % (self.mark * int(marks * (percent / 100.0)), )
+            fmt = "\r  %-10.10s: %-20.20s " + bar + " " + done
+        else:
+            bar = fmt_bar % (self.mark * marks, )
+            fmt = "  %-10.10s: %-20.20s "  + bar + " " + done
+        return fmt
+
+    def _logPkgString(self, hdr):
+        """return nice representation of the package for the log"""
+        (n,a,e,v,r) = self._dopkgtup(hdr)
+        if e == '0':
+            pkg = '%s.%s %s-%s' % (n, a, v, r)
+        else:
+            pkg = '%s.%s %s:%s-%s' % (n, a, e, v, r)
+
+        return pkg
+
+    def callback(self, what, bytes, total, h, user):
+        if what == rpm.RPMCALLBACK_TRANS_START:
+            if bytes == 6:
+                self.total_actions = total
+
+        elif what == rpm.RPMCALLBACK_TRANS_PROGRESS:
+            pass
+
+        elif what == rpm.RPMCALLBACK_TRANS_STOP:
+            pass
+
+        elif what == rpm.RPMCALLBACK_INST_OPEN_FILE:
+            self.lastmsg = None
+            hdr = None
+            if h is not None:
+                try:
+                    hdr, rpmloc = h
+                except:
+                    rpmloc = h
+                    hdr = readRpmHeader(self.ts, h)
+
+                handle = self._makeHandle(hdr)
+                fd = os.open(rpmloc, os.O_RDONLY)
+                self.callbackfilehandles[handle]=fd
+                if hdr['name'] not in self.installed_pkg_names:
+                    self.installed_pkg_names.append(hdr['name'])
+                    self.total_installed += 1
+                return fd
+            else:
+                self._localprint("No header - huh?")
+
+        elif what == rpm.RPMCALLBACK_INST_CLOSE_FILE:
+            hdr = None
+            if h is not None:
+                try:
+                    hdr, rpmloc = h
+                except:
+                    rpmloc = h
+                    hdr = readRpmHeader(self.ts, h)
+
+                handle = self._makeHandle(hdr)
+                os.close(self.callbackfilehandles[handle])
+                fd = 0
+
+                # log stuff
+                #pkgtup = self._dopkgtup(hdr)
+                self.logString.append(self._logPkgString(hdr))
+
+        elif what == rpm.RPMCALLBACK_INST_PROGRESS:
+            if h is not None:
+                percent = (self.total_installed*100L)/self.total_actions
+                if total > 0:
+                    try:
+                        hdr, rpmloc = h
+                    except:
+                        rpmloc = h
+
+                    m = re.match("(.*)-(\d+.*)-(\d+\.\d+)\.(.+)\.rpm", os.path.basename(rpmloc))
+                    if m:
+                        pkgname = m.group(1)
+                    else:
+                        pkgname = os.path.basename(rpmloc)
+                if self.output:
+                    fmt = self._makefmt(percent)
+                    msg = fmt % (self.headmsg, pkgname)
+                    if msg != self.lastmsg:
+                        self.lastmsg = msg
+
+                        msger.info(msg)
+
+                        if self.total_installed == self.total_actions:
+                            msger.raw('')
+                            msger.verbose('\n'.join(self.logString))
+
+        elif what == rpm.RPMCALLBACK_UNINST_START:
+            pass
+
+        elif what == rpm.RPMCALLBACK_UNINST_PROGRESS:
+            pass
+
+        elif what == rpm.RPMCALLBACK_UNINST_STOP:
+            self.total_removed += 1
+
+        elif what == rpm.RPMCALLBACK_REPACKAGE_START:
+            pass
+
+        elif what == rpm.RPMCALLBACK_REPACKAGE_STOP:
+            pass
+
+        elif what == rpm.RPMCALLBACK_REPACKAGE_PROGRESS:
+            pass
+
+def readRpmHeader(ts, filename):
+    """ Read an rpm header. """
+
+    fd = os.open(filename, os.O_RDONLY)
+    h = ts.hdrFromFdno(fd)
+    os.close(fd)
+    return h
+
+def splitFilename(filename):
+    """ Pass in a standard style rpm fullname
+
+        Return a name, version, release, epoch, arch, e.g.::
+            foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386
+            1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64
+    """
+
+    if filename[-4:] == '.rpm':
+        filename = filename[:-4]
+
+    archIndex = filename.rfind('.')
+    arch = filename[archIndex+1:]
+
+    relIndex = filename[:archIndex].rfind('-')
+    rel = filename[relIndex+1:archIndex]
+
+    verIndex = filename[:relIndex].rfind('-')
+    ver = filename[verIndex+1:relIndex]
+
+    epochIndex = filename.find(':')
+    if epochIndex == -1:
+        epoch = ''
+    else:
+        epoch = filename[:epochIndex]
+
+    name = filename[epochIndex + 1:verIndex]
+    return name, ver, rel, epoch, arch
+
+def getCanonX86Arch(arch):
+    #
+    if arch == "i586":
+        f = open("/proc/cpuinfo", "r")
+        lines = f.readlines()
+        f.close()
+        for line in lines:
+            if line.startswith("model name") and line.find("Geode(TM)") != -1:
+                return "geode"
+        return arch
+    # only athlon vs i686 isn't handled with uname currently
+    if arch != "i686":
+        return arch
+
+    # if we're i686 and AuthenticAMD, then we should be an athlon
+    f = open("/proc/cpuinfo", "r")
+    lines = f.readlines()
+    f.close()
+    for line in lines:
+        if line.startswith("vendor") and line.find("AuthenticAMD") != -1:
+            return "athlon"
+        # i686 doesn't guarantee cmov, but we depend on it
+        elif line.startswith("flags") and line.find("cmov") == -1:
+            return "i586"
+
+    return arch
+
+def getCanonX86_64Arch(arch):
+    if arch != "x86_64":
+        return arch
+
+    vendor = None
+    f = open("/proc/cpuinfo", "r")
+    lines = f.readlines()
+    f.close()
+    for line in lines:
+        if line.startswith("vendor_id"):
+            vendor = line.split(':')[1]
+            break
+    if vendor is None:
+        return arch
+
+    if vendor.find("Authentic AMD") != -1 or vendor.find("AuthenticAMD") != -1:
+        return "amd64"
+    if vendor.find("GenuineIntel") != -1:
+        return "ia32e"
+    return arch
+
+def getCanonArch():
+    arch = os.uname()[4]
+
+    if (len(arch) == 4 and arch[0] == "i" and arch[2:4] == "86"):
+        return getCanonX86Arch(arch)
+
+    if arch == "x86_64":
+        return getCanonX86_64Arch(arch)
+
+    return arch
+
+# Copy from libsatsolver:poolarch.c, with cleanup
+archPolicies = {
+    "x86_64":       "x86_64:i686:i586:i486:i386",
+    "i686":         "i686:i586:i486:i386",
+    "i586":         "i586:i486:i386",
+    "ia64":         "ia64:i686:i586:i486:i386",
+    "armv7tnhl":    "armv7tnhl:armv7thl:armv7nhl:armv7hl",
+    "armv7thl":     "armv7thl:armv7hl",
+    "armv7nhl":     "armv7nhl:armv7hl",
+    "armv7hl":      "armv7hl",
+    "armv7l":       "armv7l:armv6l:armv5tejl:armv5tel:armv5l:armv4tl:armv4l:armv3l",
+    "armv6l":       "armv6l:armv5tejl:armv5tel:armv5l:armv4tl:armv4l:armv3l",
+    "armv5tejl":    "armv5tejl:armv5tel:armv5l:armv4tl:armv4l:armv3l",
+    "armv5tel":     "armv5tel:armv5l:armv4tl:armv4l:armv3l",
+    "armv5l":       "armv5l:armv4tl:armv4l:armv3l",
+}
+
+# dict mapping arch -> ( multicompat, best personality, biarch personality )
+multilibArches = {
+    "x86_64":  ( "athlon", "x86_64", "athlon" ),
+}
+
+# from yumUtils.py
+arches = {
+    # ia32
+    "athlon": "i686",
+    "i686": "i586",
+    "geode": "i586",
+    "i586": "i486",
+    "i486": "i386",
+    "i386": "noarch",
+
+    # amd64
+    "x86_64": "athlon",
+    "amd64": "x86_64",
+    "ia32e": "x86_64",
+
+    # arm
+    "armv7tnhl": "armv7nhl",
+    "armv7nhl": "armv7hl",
+    "armv7hl": "noarch",
+    "armv7l": "armv6l",
+    "armv6l": "armv5tejl",
+    "armv5tejl": "armv5tel",
+    "armv5tel": "noarch",
+
+    #itanium
+    "ia64": "noarch",
+}
+
+def isMultiLibArch(arch=None):
+    """returns true if arch is a multilib arch, false if not"""
+    if arch is None:
+        arch = getCanonArch()
+
+    if not arches.has_key(arch): # or we could check if it is noarch
+        return False
+
+    if multilibArches.has_key(arch):
+        return True
+
+    if multilibArches.has_key(arches[arch]):
+        return True
+
+    return False
+
+def getBaseArch():
+    myarch = getCanonArch()
+    if not arches.has_key(myarch):
+        return myarch
+
+    if isMultiLibArch(arch=myarch):
+        if multilibArches.has_key(myarch):
+            return myarch
+        else:
+            return arches[myarch]
+
+    if arches.has_key(myarch):
+        basearch = myarch
+        value = arches[basearch]
+        while value != 'noarch':
+            basearch = value
+            value = arches[basearch]
+
+        return basearch
+
+def checkRpmIntegrity(bin_rpm, package):
+    return runner.quiet([bin_rpm, "-K", "--nosignature", package])
+
+def checkSig(ts, package):
+    """ Takes a transaction set and a package, check it's sigs,
+        return 0 if they are all fine
+        return 1 if the gpg key can't be found
+        return 2 if the header is in someway damaged
+        return 3 if the key is not trusted
+        return 4 if the pkg is not gpg or pgp signed
+    """
+
+    value = 0
+    currentflags = ts.setVSFlags(0)
+    fdno = os.open(package, os.O_RDONLY)
+    try:
+        hdr = ts.hdrFromFdno(fdno)
+
+    except rpm.error, e:
+        if str(e) == "public key not availaiable":
+            value = 1
+        if str(e) == "public key not available":
+            value = 1
+        if str(e) == "public key not trusted":
+            value = 3
+        if str(e) == "error reading package header":
+            value = 2
+    else:
+        error, siginfo = getSigInfo(hdr)
+        if error == 101:
+            os.close(fdno)
+            del hdr
+            value = 4
+        else:
+            del hdr
+
+    try:
+        os.close(fdno)
+    except OSError:
+        pass
+
+    ts.setVSFlags(currentflags) # put things back like they were before
+    return value
+
+def getSigInfo(hdr):
+    """ checks signature from an hdr hand back signature information and/or
+        an error code
+    """
+
+    import locale
+    locale.setlocale(locale.LC_ALL, 'C')
+
+    string = '%|DSAHEADER?{%{DSAHEADER:pgpsig}}:{%|RSAHEADER?{%{RSAHEADER:pgpsig}}:{%|SIGGPG?{%{SIGGPG:pgpsig}}:{%|SIGPGP?{%{SIGPGP:pgpsig}}:{(none)}|}|}|}|'
+    siginfo = hdr.sprintf(string)
+    if siginfo != '(none)':
+        error = 0
+        sigtype, sigdate, sigid = siginfo.split(',')
+    else:
+        error = 101
+        sigtype = 'MD5'
+        sigdate = 'None'
+        sigid = 'None'
+
+    infotuple = (sigtype, sigdate, sigid)
+    return error, infotuple
+
+def checkRepositoryEULA(name, repo):
+    """ This function is to check the EULA file if provided.
+        return True: no EULA or accepted
+        return False: user declined the EULA
+    """
+
+    import tempfile
+    import shutil
+    import urlparse
+    import urllib2 as u2
+    import httplib
+    from mic.utils.errors import CreatorError
+
+    def _check_and_download_url(u2opener, url, savepath):
+        try:
+            if u2opener:
+                f = u2opener.open(url)
+            else:
+                f = u2.urlopen(url)
+        except u2.HTTPError, httperror:
+            if httperror.code in (404, 503):
+                return None
+            else:
+                raise CreatorError(httperror)
+        except OSError, oserr:
+            if oserr.errno == 2:
+                return None
+            else:
+                raise CreatorError(oserr)
+        except IOError, oserr:
+            if hasattr(oserr, "reason") and oserr.reason.errno == 2:
+                return None
+            else:
+                raise CreatorError(oserr)
+        except u2.URLError, err:
+            raise CreatorError(err)
+        except httplib.HTTPException, e:
+            raise CreatorError(e)
+
+        # save to file
+        licf = open(savepath, "w")
+        licf.write(f.read())
+        licf.close()
+        f.close()
+
+        return savepath
+
+    def _pager_file(savepath):
+
+        if os.path.splitext(savepath)[1].upper() in ('.HTM', '.HTML'):
+            pagers = ('w3m', 'links', 'lynx', 'less', 'more')
+        else:
+            pagers = ('less', 'more')
+
+        file_showed = False
+        for pager in pagers:
+            cmd = "%s %s" % (pager, savepath)
+            try:
+                os.system(cmd)
+            except OSError:
+                continue
+            else:
+                file_showed = True
+                break
+
+        if not file_showed:
+            f = open(savepath)
+            msger.raw(f.read())
+            f.close()
+            msger.pause()
+
+    # when proxy needed, make urllib2 follow it
+    proxy = repo.proxy
+    proxy_username = repo.proxy_username
+    proxy_password = repo.proxy_password
+
+    if not proxy:
+        proxy = get_proxy_for(repo.baseurl[0])
+
+    handlers = []
+    auth_handler = u2.HTTPBasicAuthHandler(u2.HTTPPasswordMgrWithDefaultRealm())
+    u2opener = None
+    if proxy:
+        if proxy_username:
+            proxy_netloc = urlparse.urlsplit(proxy).netloc
+            if proxy_password:
+                proxy_url = 'http://%s:%s@%s' % (proxy_username, proxy_password, proxy_netloc)
+            else:
+                proxy_url = 'http://%s@%s' % (proxy_username, proxy_netloc)
+        else:
+            proxy_url = proxy
+
+        proxy_support = u2.ProxyHandler({'http': proxy_url,
+                                         'https': proxy_url,
+                                         'ftp': proxy_url})
+        handlers.append(proxy_support)
+
+    # download all remote files to one temp dir
+    baseurl = None
+    repo_lic_dir = tempfile.mkdtemp(prefix = 'repolic')
+
+    for url in repo.baseurl:
+        tmphandlers = handlers[:]
+
+        (scheme, host, path, parm, query, frag) = urlparse.urlparse(url.rstrip('/') + '/')
+        if scheme not in ("http", "https", "ftp", "ftps", "file"):
+            raise CreatorError("Error: invalid url %s" % url)
+
+        if '@' in host:
+            try:
+                user_pass, host = host.split('@', 1)
+                if ':' in user_pass:
+                    user, password = user_pass.split(':', 1)
+            except ValueError, e:
+                raise CreatorError('Bad URL: %s' % url)
+
+            msger.verbose("adding HTTP auth: %s, XXXXXXXX" %(user))
+            auth_handler.add_password(None, host, user, password)
+            tmphandlers.append(auth_handler)
+            url = scheme + "://" + host + path + parm + query + frag
+
+        if tmphandlers:
+            u2opener = u2.build_opener(*tmphandlers)
+
+        # try to download
+        repo_eula_url = urlparse.urljoin(url, "LICENSE.txt")
+        repo_eula_path = _check_and_download_url(
+                                u2opener,
+                                repo_eula_url,
+                                os.path.join(repo_lic_dir, repo.id + '_LICENSE.txt'))
+        if repo_eula_path:
+            # found
+            baseurl = url
+            break
+
+    if not baseurl:
+        shutil.rmtree(repo_lic_dir) #cleanup
+        return True
+
+    # show the license file
+    msger.info('For the software packages in this yum repo:')
+    msger.info('    %s: %s' % (name, baseurl))
+    msger.info('There is an "End User License Agreement" file that need to be checked.')
+    msger.info('Please read the terms and conditions outlined in it and answer the followed qustions.')
+    msger.pause()
+
+    _pager_file(repo_eula_path)
+
+    # Asking for the "Accept/Decline"
+    if not msger.ask('Would you agree to the terms and conditions outlined in the above End User License Agreement?'):
+        msger.warning('Will not install pkgs from this repo.')
+        shutil.rmtree(repo_lic_dir) #cleanup
+        return False
+
+    # try to find support_info.html for extra infomation
+    repo_info_url = urlparse.urljoin(baseurl, "support_info.html")
+    repo_info_path = _check_and_download_url(
+                            u2opener,
+                            repo_info_url,
+                            os.path.join(repo_lic_dir, repo.id + '_support_info.html'))
+    if repo_info_path:
+        msger.info('There is one more file in the repo for additional support information, please read it')
+        msger.pause()
+        _pager_file(repo_info_path)
+
+    #cleanup
+    shutil.rmtree(repo_lic_dir)
+    return True
diff --git a/scripts/lib/mic/utils/runner.py b/scripts/lib/mic/utils/runner.py
new file mode 100644
index 0000000000..fded3c93fa
--- /dev/null
+++ b/scripts/lib/mic/utils/runner.py
@@ -0,0 +1,109 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os
+import subprocess
+
+from mic import msger
+
+def runtool(cmdln_or_args, catch=1):
+    """ wrapper for most of the subprocess calls
+    input:
+        cmdln_or_args: can be both args and cmdln str (shell=True)
+        catch: 0, quitely run
+               1, only STDOUT
+               2, only STDERR
+               3, both STDOUT and STDERR
+    return:
+        (rc, output)
+        if catch==0: the output will always None
+    """
+
+    if catch not in (0, 1, 2, 3):
+        # invalid catch selection, will cause exception, that's good
+        return None
+
+    if isinstance(cmdln_or_args, list):
+        cmd = cmdln_or_args[0]
+        shell = False
+    else:
+        import shlex
+        cmd = shlex.split(cmdln_or_args)[0]
+        shell = True
+
+    if catch != 3:
+        dev_null = os.open("/dev/null", os.O_WRONLY)
+
+    if catch == 0:
+        sout = dev_null
+        serr = dev_null
+    elif catch == 1:
+        sout = subprocess.PIPE
+        serr = dev_null
+    elif catch == 2:
+        sout = dev_null
+        serr = subprocess.PIPE
+    elif catch == 3:
+        sout = subprocess.PIPE
+        serr = subprocess.STDOUT
+
+    try:
+        p = subprocess.Popen(cmdln_or_args, stdout=sout,
+                             stderr=serr, shell=shell)
+        (sout, serr) = p.communicate()
+        # combine stdout and stderr, filter None out
+        out = ''.join(filter(None, [sout, serr]))
+    except OSError, e:
+        if e.errno == 2:
+            # [Errno 2] No such file or directory
+            msger.error('Cannot run command: %s, lost dependency?' % cmd)
+        else:
+            raise # relay
+    finally:
+        if catch != 3:
+            os.close(dev_null)
+
+    return (p.returncode, out)
+
+def show(cmdln_or_args):
+    # show all the message using msger.verbose
+
+    rc, out = runtool(cmdln_or_args, catch=3)
+
+    if isinstance(cmdln_or_args, list):
+        cmd = ' '.join(cmdln_or_args)
+    else:
+        cmd = cmdln_or_args
+
+    msg =  'running command: "%s"' % cmd
+    if out: out = out.strip()
+    if out:
+        msg += ', with output::'
+        msg += '\n  +----------------'
+        for line in out.splitlines():
+            msg += '\n  | %s' % line
+        msg += '\n  +----------------'
+
+    msger.verbose(msg)
+    return rc
+
+def outs(cmdln_or_args, catch=1):
+    # get the outputs of tools
+    return runtool(cmdln_or_args, catch)[1].strip()
+
+def quiet(cmdln_or_args):
+    return runtool(cmdln_or_args, catch=0)[0]