10 files changed, 3309 insertions, 0 deletions

diff --git a/scripts/lib/wic/utils/__init__.py b/scripts/lib/wic/utils/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/scripts/lib/wic/utils/__init__.py
diff --git a/scripts/lib/wic/utils/cmdln.py b/scripts/lib/wic/utils/cmdln.py
new file mode 100644
index 0000000000..b099473ee4
--- /dev/null
+++ b/scripts/lib/wic/utils/cmdln.py
@@ -0,0 +1,1586 @@
+#!/usr/bin/env python
+# Copyright (c) 2002-2007 ActiveState Software Inc.
+# License: MIT (see LICENSE.txt for license details)
+# Author:  Trent Mick
+# Home:    http://trentm.com/projects/cmdln/
+
+"""An improvement on Python's standard cmd.py module.
+
+As with cmd.py, this module provides "a simple framework for writing
+line-oriented command intepreters."  This module provides a 'RawCmdln'
+class that fixes some design flaws in cmd.Cmd, making it more scalable
+and nicer to use for good 'cvs'- or 'svn'-style command line interfaces
+or simple shells.  And it provides a 'Cmdln' class that add
+optparse-based option processing. Basically you use it like this:
+
+    import cmdln
+
+    class MySVN(cmdln.Cmdln):
+        name = "svn"
+
+        @cmdln.alias('stat', 'st')
+        @cmdln.option('-v', '--verbose', action='store_true'
+                      help='print verbose information')
+        def do_status(self, subcmd, opts, *paths):
+            print "handle 'svn status' command"
+
+        #...
+
+    if __name__ == "__main__":
+        shell = MySVN()
+        retval = shell.main()
+        sys.exit(retval)
+
+See the README.txt or <http://trentm.com/projects/cmdln/> for more
+details.
+"""
+
+__version_info__ = (1, 1, 2)
+__version__ = '.'.join(map(str, __version_info__))
+
+import os
+import sys
+import re
+import cmd
+import optparse
+from pprint import pprint
+import sys
+
+
+
+
+#---- globals
+
+LOOP_ALWAYS, LOOP_NEVER, LOOP_IF_EMPTY = range(3)
+
+# An unspecified optional argument when None is a meaningful value.
+_NOT_SPECIFIED = ("Not", "Specified")
+
+# Pattern to match a TypeError message from a call that
+# failed because of incorrect number of arguments (see
+# Python/getargs.c).
+_INCORRECT_NUM_ARGS_RE = re.compile(
+    r"(takes [\w ]+ )(\d+)( arguments? \()(\d+)( given\))")
+
+
+
+#---- exceptions
+
+class CmdlnError(Exception):
+    """A cmdln.py usage error."""
+    def __init__(self, msg):
+        self.msg = msg
+    def __str__(self):
+        return self.msg
+
+class CmdlnUserError(Exception):
+    """An error by a user of a cmdln-based tool/shell."""
+    pass
+
+
+
+#---- public methods and classes
+
+def alias(*aliases):
+    """Decorator to add aliases for Cmdln.do_* command handlers.
+    
+    Example:
+        class MyShell(cmdln.Cmdln):
+            @cmdln.alias("!", "sh")
+            def do_shell(self, argv):
+                #...implement 'shell' command
+    """
+    def decorate(f):
+        if not hasattr(f, "aliases"):
+            f.aliases = []
+        f.aliases += aliases
+        return f
+    return decorate
+
+
+class RawCmdln(cmd.Cmd):
+    """An improved (on cmd.Cmd) framework for building multi-subcommand
+    scripts (think "svn" & "cvs") and simple shells (think "pdb" and
+    "gdb").
+
+    A simple example:
+
+        import cmdln
+
+        class MySVN(cmdln.RawCmdln):
+            name = "svn"
+
+            @cmdln.aliases('stat', 'st')
+            def do_status(self, argv):
+                print "handle 'svn status' command"
+
+        if __name__ == "__main__":
+            shell = MySVN()
+            retval = shell.main()
+            sys.exit(retval)
+
+    See <http://trentm.com/projects/cmdln> for more information.
+    """
+    name = None      # if unset, defaults basename(sys.argv[0])
+    prompt = None    # if unset, defaults to self.name+"> "
+    version = None   # if set, default top-level options include --version
+
+    # Default messages for some 'help' command error cases.
+    # They are interpolated with one arg: the command.
+    nohelp = "no help on '%s'"
+    unknowncmd = "unknown command: '%s'"
+
+    helpindent = '' # string with which to indent help output
+
+    def __init__(self, completekey='tab', 
+                 stdin=None, stdout=None, stderr=None):
+        """Cmdln(completekey='tab', stdin=None, stdout=None, stderr=None)
+
+        The optional argument 'completekey' is the readline name of a
+        completion key; it defaults to the Tab key. If completekey is
+        not None and the readline module is available, command completion
+        is done automatically.
+        
+        The optional arguments 'stdin', 'stdout' and 'stderr' specify
+        alternate input, output and error output file objects; if not
+        specified, sys.* are used.
+        
+        If 'stdout' but not 'stderr' is specified, stdout is used for
+        error output. This is to provide least surprise for users used
+        to only the 'stdin' and 'stdout' options with cmd.Cmd.
+        """
+        import sys
+        if self.name is None:
+            self.name = os.path.basename(sys.argv[0])
+        if self.prompt is None:
+            self.prompt = self.name+"> "
+        self._name_str = self._str(self.name)
+        self._prompt_str = self._str(self.prompt)
+        if stdin is not None:
+            self.stdin = stdin
+        else:
+            self.stdin = sys.stdin
+        if stdout is not None:
+            self.stdout = stdout
+        else:
+            self.stdout = sys.stdout
+        if stderr is not None:
+            self.stderr = stderr
+        elif stdout is not None:
+            self.stderr = stdout
+        else:
+            self.stderr = sys.stderr
+        self.cmdqueue = []
+        self.completekey = completekey
+        self.cmdlooping = False
+
+    def get_optparser(self):
+        """Hook for subclasses to set the option parser for the
+        top-level command/shell.
+
+        This option parser is used retrieved and used by `.main()' to
+        handle top-level options.
+
+        The default implements a single '-h|--help' option. Sub-classes
+        can return None to have no options at the top-level. Typically
+        an instance of CmdlnOptionParser should be returned.
+        """
+        version = (self.version is not None 
+                    and "%s %s" % (self._name_str, self.version)
+                    or None)
+        return CmdlnOptionParser(self, version=version)
+
+    def postoptparse(self):
+        """Hook method executed just after `.main()' parses top-level
+        options.
+
+        When called `self.options' holds the results of the option parse.
+        """
+        pass
+
+    def main(self, argv=None, loop=LOOP_NEVER):
+        """A possible mainline handler for a script, like so:
+
+            import cmdln
+            class MyCmd(cmdln.Cmdln):
+                name = "mycmd"
+                ...
+            
+            if __name__ == "__main__":
+                MyCmd().main()
+
+        By default this will use sys.argv to issue a single command to
+        'MyCmd', then exit. The 'loop' argument can be use to control
+        interactive shell behaviour.
+        
+        Arguments:
+            "argv" (optional, default sys.argv) is the command to run.
+                It must be a sequence, where the first element is the
+                command name and subsequent elements the args for that
+                command.
+            "loop" (optional, default LOOP_NEVER) is a constant
+                indicating if a command loop should be started (i.e. an
+                interactive shell). Valid values (constants on this module):
+                    LOOP_ALWAYS     start loop and run "argv", if any
+                    LOOP_NEVER      run "argv" (or .emptyline()) and exit
+                    LOOP_IF_EMPTY   run "argv", if given, and exit;
+                                    otherwise, start loop
+        """
+        if argv is None:
+            import sys
+            argv = sys.argv
+        else:
+            argv = argv[:] # don't modify caller's list
+
+        self.optparser = self.get_optparser()
+        if self.optparser: # i.e. optparser=None means don't process for opts
+            try:
+                self.options, args = self.optparser.parse_args(argv[1:])
+            except CmdlnUserError, ex:
+                msg = "%s: %s\nTry '%s help' for info.\n"\
+                      % (self.name, ex, self.name)
+                self.stderr.write(self._str(msg))
+                self.stderr.flush()
+                return 1
+            except StopOptionProcessing, ex:
+                return 0
+        else:
+            self.options, args = None, argv[1:]
+        self.postoptparse()
+
+        if loop == LOOP_ALWAYS:
+            if args:
+                self.cmdqueue.append(args)
+            return self.cmdloop()
+        elif loop == LOOP_NEVER:
+            if args:
+                return self.cmd(args)
+            else:
+                return self.emptyline()
+        elif loop == LOOP_IF_EMPTY:
+            if args:
+                return self.cmd(args)
+            else:
+                return self.cmdloop()
+
+    def cmd(self, argv):
+        """Run one command and exit.
+        
+            "argv" is the arglist for the command to run. argv[0] is the
+                command to run. If argv is an empty list then the
+                'emptyline' handler is run.
+
+        Returns the return value from the command handler.
+        """
+        assert isinstance(argv, (list, tuple)), \
+                "'argv' is not a sequence: %r" % argv
+        retval = None
+        try:
+            argv = self.precmd(argv)
+            retval = self.onecmd(argv)
+            self.postcmd(argv)
+        except:
+            if not self.cmdexc(argv):
+                raise
+            retval = 1
+        return retval
+
+    def _str(self, s):
+        """Safely convert the given str/unicode to a string for printing."""
+        try:
+            return str(s)
+        except UnicodeError:
+            #XXX What is the proper encoding to use here? 'utf-8' seems
+            #    to work better than "getdefaultencoding" (usually
+            #    'ascii'), on OS X at least.
+            #import sys
+            #return s.encode(sys.getdefaultencoding(), "replace")
+            return s.encode("utf-8", "replace")
+
+    def cmdloop(self, intro=None):
+        """Repeatedly issue a prompt, accept input, parse into an argv, and
+        dispatch (via .precmd(), .onecmd() and .postcmd()), passing them
+        the argv. In other words, start a shell.
+        
+            "intro" (optional) is a introductory message to print when
+                starting the command loop. This overrides the class
+                "intro" attribute, if any.
+        """
+        self.cmdlooping = True
+        self.preloop()
+        if self.use_rawinput and self.completekey:
+            try:
+                import readline
+                self.old_completer = readline.get_completer()
+                readline.set_completer(self.complete)
+                readline.parse_and_bind(self.completekey+": complete")
+            except ImportError:
+                pass
+        try:
+            if intro is None:
+                intro = self.intro
+            if intro:
+                intro_str = self._str(intro)
+                self.stdout.write(intro_str+'\n')
+            self.stop = False
+            retval = None
+            while not self.stop:
+                if self.cmdqueue:
+                    argv = self.cmdqueue.pop(0)
+                    assert isinstance(argv, (list, tuple)), \
+                            "item on 'cmdqueue' is not a sequence: %r" % argv
+                else:
+                    if self.use_rawinput:
+                        try:
+                            line = raw_input(self._prompt_str)
+                        except EOFError:
+                            line = 'EOF'
+                    else:
+                        self.stdout.write(self._prompt_str)
+                        self.stdout.flush()
+                        line = self.stdin.readline()
+                        if not len(line):
+                            line = 'EOF'
+                        else:
+                            line = line[:-1] # chop '\n'
+                    argv = line2argv(line)
+                try:
+                    argv = self.precmd(argv)
+                    retval = self.onecmd(argv)
+                    self.postcmd(argv)
+                except:
+                    if not self.cmdexc(argv):
+                        raise
+                    retval = 1
+                self.lastretval = retval
+            self.postloop()
+        finally:
+            if self.use_rawinput and self.completekey:
+                try:
+                    import readline
+                    readline.set_completer(self.old_completer)
+                except ImportError:
+                    pass
+        self.cmdlooping = False
+        return retval
+
+    def precmd(self, argv):
+        """Hook method executed just before the command argv is
+        interpreted, but after the input prompt is generated and issued.
+
+            "argv" is the cmd to run.
+            
+        Returns an argv to run (i.e. this method can modify the command
+        to run).
+        """
+        return argv
+
+    def postcmd(self, argv):
+        """Hook method executed just after a command dispatch is finished.
+        
+            "argv" is the command that was run.
+        """
+        pass
+
+    def cmdexc(self, argv):
+        """Called if an exception is raised in any of precmd(), onecmd(),
+        or postcmd(). If True is returned, the exception is deemed to have
+        been dealt with. Otherwise, the exception is re-raised.
+
+        The default implementation handles CmdlnUserError's, which
+        typically correspond to user error in calling commands (as
+        opposed to programmer error in the design of the script using
+        cmdln.py).
+        """
+        import sys
+        type, exc, traceback = sys.exc_info()
+        if isinstance(exc, CmdlnUserError):
+            msg = "%s %s: %s\nTry '%s help %s' for info.\n"\
+                  % (self.name, argv[0], exc, self.name, argv[0])
+            self.stderr.write(self._str(msg))
+            self.stderr.flush()
+            return True
+
+    def onecmd(self, argv):
+        if not argv:
+            return self.emptyline()
+        self.lastcmd = argv
+        cmdname = self._get_canonical_cmd_name(argv[0])
+        if cmdname:
+            handler = self._get_cmd_handler(cmdname)
+            if handler:
+                return self._dispatch_cmd(handler, argv)
+        return self.default(argv)
+
+    def _dispatch_cmd(self, handler, argv):
+        return handler(argv)
+
+    def default(self, argv):
+        """Hook called to handle a command for which there is no handler.
+
+            "argv" is the command and arguments to run.
+        
+        The default implementation writes and error message to stderr
+        and returns an error exit status.
+
+        Returns a numeric command exit status.
+        """
+        errmsg = self._str(self.unknowncmd % (argv[0],))
+        if self.cmdlooping:
+            self.stderr.write(errmsg+"\n")
+        else:
+            self.stderr.write("%s: %s\nTry '%s help' for info.\n"
+                              % (self._name_str, errmsg, self._name_str))
+        self.stderr.flush()
+        return 1
+
+    def parseline(self, line):
+        # This is used by Cmd.complete (readline completer function) to
+        # massage the current line buffer before completion processing.
+        # We override to drop special '!' handling.
+        line = line.strip()
+        if not line:
+            return None, None, line
+        elif line[0] == '?':
+            line = 'help ' + line[1:]
+        i, n = 0, len(line)
+        while i < n and line[i] in self.identchars: i = i+1
+        cmd, arg = line[:i], line[i:].strip()
+        return cmd, arg, line
+
+    def helpdefault(self, cmd, known):
+        """Hook called to handle help on a command for which there is no
+        help handler.
+
+            "cmd" is the command name on which help was requested.
+            "known" is a boolean indicating if this command is known
+                (i.e. if there is a handler for it).
+        
+        Returns a return code.
+        """
+        if known:
+            msg = self._str(self.nohelp % (cmd,))
+            if self.cmdlooping:
+                self.stderr.write(msg + '\n')
+            else:
+                self.stderr.write("%s: %s\n" % (self.name, msg))
+        else:
+            msg = self.unknowncmd % (cmd,)
+            if self.cmdlooping:
+                self.stderr.write(msg + '\n')
+            else:
+                self.stderr.write("%s: %s\n"
+                                  "Try '%s help' for info.\n"
+                                  % (self.name, msg, self.name))
+        self.stderr.flush()
+        return 1
+
+    def do_help(self, argv):
+        """${cmd_name}: give detailed help on a specific sub-command
+
+        Usage:
+            ${name} help [COMMAND]
+        """
+        if len(argv) > 1: # asking for help on a particular command
+            doc = None
+            cmdname = self._get_canonical_cmd_name(argv[1]) or argv[1]
+            if not cmdname:
+                return self.helpdefault(argv[1], False)
+            else:
+                helpfunc = getattr(self, "help_"+cmdname, None)
+                if helpfunc:
+                    doc = helpfunc()
+                else:
+                    handler = self._get_cmd_handler(cmdname)
+                    if handler:
+                        doc = handler.__doc__
+                    if doc is None:
+                        return self.helpdefault(argv[1], handler != None)
+        else: # bare "help" command
+            doc = self.__class__.__doc__  # try class docstring
+            if doc is None:
+                # Try to provide some reasonable useful default help.
+                if self.cmdlooping: prefix = ""
+                else:               prefix = self.name+' '
+                doc = """Usage:
+                    %sCOMMAND [ARGS...]
+                    %shelp [COMMAND]
+
+                ${option_list}
+                ${command_list}
+                ${help_list}
+                """ % (prefix, prefix)
+            cmdname = None
+
+        if doc: # *do* have help content, massage and print that
+            doc = self._help_reindent(doc)
+            doc = self._help_preprocess(doc, cmdname)
+            doc = doc.rstrip() + '\n' # trim down trailing space
+            self.stdout.write(self._str(doc))
+            self.stdout.flush()
+    do_help.aliases = ["?"]
+
+    def _help_reindent(self, help, indent=None):
+        """Hook to re-indent help strings before writing to stdout.
+
+            "help" is the help content to re-indent
+            "indent" is a string with which to indent each line of the
+                help content after normalizing. If unspecified or None
+                then the default is use: the 'self.helpindent' class
+                attribute. By default this is the empty string, i.e.
+                no indentation.
+
+        By default, all common leading whitespace is removed and then
+        the lot is indented by 'self.helpindent'. When calculating the
+        common leading whitespace the first line is ignored -- hence
+        help content for Conan can be written as follows and have the
+        expected indentation:
+
+            def do_crush(self, ...):
+                '''${cmd_name}: crush your enemies, see them driven before you...
+
+                c.f. Conan the Barbarian'''
+        """
+        if indent is None:
+            indent = self.helpindent
+        lines = help.splitlines(0)
+        _dedentlines(lines, skip_first_line=True)
+        lines = [(indent+line).rstrip() for line in lines]
+        return '\n'.join(lines)
+
+    def _help_preprocess(self, help, cmdname):
+        """Hook to preprocess a help string before writing to stdout.
+
+            "help" is the help string to process.
+            "cmdname" is the canonical sub-command name for which help
+                is being given, or None if the help is not specific to a
+                command.
+
+        By default the following template variables are interpolated in
+        help content. (Note: these are similar to Python 2.4's
+        string.Template interpolation but not quite.)
+
+        ${name}
+            The tool's/shell's name, i.e. 'self.name'.
+        ${option_list}
+            A formatted table of options for this shell/tool.
+        ${command_list}
+            A formatted table of available sub-commands.
+        ${help_list}
+            A formatted table of additional help topics (i.e. 'help_*'
+            methods with no matching 'do_*' method).
+        ${cmd_name}
+            The name (and aliases) for this sub-command formatted as:
+            "NAME (ALIAS1, ALIAS2, ...)".
+        ${cmd_usage}
+            A formatted usage block inferred from the command function
+            signature.
+        ${cmd_option_list}
+            A formatted table of options for this sub-command. (This is
+            only available for commands using the optparse integration,
+            i.e.  using @cmdln.option decorators or manually setting the
+            'optparser' attribute on the 'do_*' method.)
+
+        Returns the processed help. 
+        """
+        preprocessors = {
+            "${name}":            self._help_preprocess_name,
+            "${option_list}":     self._help_preprocess_option_list,
+            "${command_list}":    self._help_preprocess_command_list,
+            "${help_list}":       self._help_preprocess_help_list,
+            "${cmd_name}":        self._help_preprocess_cmd_name,
+            "${cmd_usage}":       self._help_preprocess_cmd_usage,
+            "${cmd_option_list}": self._help_preprocess_cmd_option_list,
+        }
+
+        for marker, preprocessor in preprocessors.items():
+            if marker in help:
+                help = preprocessor(help, cmdname)
+        return help
+
+    def _help_preprocess_name(self, help, cmdname=None):
+        return help.replace("${name}", self.name)
+
+    def _help_preprocess_option_list(self, help, cmdname=None):
+        marker = "${option_list}"
+        indent, indent_width = _get_indent(marker, help)
+        suffix = _get_trailing_whitespace(marker, help)
+
+        if self.optparser:
+            # Setup formatting options and format.
+            # - Indentation of 4 is better than optparse default of 2.
+            #   C.f. Damian Conway's discussion of this in Perl Best
+            #   Practices.
+            self.optparser.formatter.indent_increment = 4
+            self.optparser.formatter.current_indent = indent_width
+            block = self.optparser.format_option_help() + '\n'
+        else:
+            block = ""
+            
+        help = help.replace(indent+marker+suffix, block, 1)
+        return help
+
+
+    def _help_preprocess_command_list(self, help, cmdname=None):
+        marker = "${command_list}"
+        indent, indent_width = _get_indent(marker, help)
+        suffix = _get_trailing_whitespace(marker, help)
+
+        # Find any aliases for commands.
+        token2canonical = self._get_canonical_map()
+        aliases = {}
+        for token, cmdname in token2canonical.items():
+            if token == cmdname: continue
+            aliases.setdefault(cmdname, []).append(token)
+
+        # Get the list of (non-hidden) commands and their
+        # documentation, if any.
+        cmdnames = {} # use a dict to strip duplicates
+        for attr in self.get_names():
+            if attr.startswith("do_"):
+                cmdnames[attr[3:]] = True
+        cmdnames = cmdnames.keys()
+        cmdnames.sort()
+        linedata = []
+        for cmdname in cmdnames:
+            if aliases.get(cmdname):
+                a = aliases[cmdname]
+                a.sort()
+                cmdstr = "%s (%s)" % (cmdname, ", ".join(a))
+            else:
+                cmdstr = cmdname
+            doc = None
+            try:
+                helpfunc = getattr(self, 'help_'+cmdname)
+            except AttributeError:
+                handler = self._get_cmd_handler(cmdname)
+                if handler:
+                    doc = handler.__doc__
+            else:
+                doc = helpfunc()
+                
+            # Strip "${cmd_name}: " from the start of a command's doc. Best
+            # practice dictates that command help strings begin with this, but
+            # it isn't at all wanted for the command list.
+            to_strip = "${cmd_name}:"
+            if doc and doc.startswith(to_strip):
+                #log.debug("stripping %r from start of %s's help string",
+                #          to_strip, cmdname)
+                doc = doc[len(to_strip):].lstrip()
+            linedata.append( (cmdstr, doc) )
+
+        if linedata:
+            subindent = indent + ' '*4
+            lines = _format_linedata(linedata, subindent, indent_width+4)
+            block = indent + "Commands:\n" \
+                    + '\n'.join(lines) + "\n\n"
+            help = help.replace(indent+marker+suffix, block, 1)
+        return help
+
+    def _gen_names_and_attrs(self):
+        # Inheritance says we have to look in class and
+        # base classes; order is not important.
+        names = []
+        classes = [self.__class__]
+        while classes:
+            aclass = classes.pop(0)
+            if aclass.__bases__:
+                classes = classes + list(aclass.__bases__)
+            for name in dir(aclass):
+                yield (name, getattr(aclass, name))
+
+    def _help_preprocess_help_list(self, help, cmdname=None):
+        marker = "${help_list}"
+        indent, indent_width = _get_indent(marker, help)
+        suffix = _get_trailing_whitespace(marker, help)
+
+        # Determine the additional help topics, if any.
+        helpnames = {}
+        token2cmdname = self._get_canonical_map()
+        for attrname, attr in self._gen_names_and_attrs():
+            if not attrname.startswith("help_"): continue
+            helpname = attrname[5:]
+            if helpname not in token2cmdname:
+                helpnames[helpname] = attr
+
+        if helpnames:
+            linedata = [(n, a.__doc__ or "") for n, a in helpnames.items()]
+            linedata.sort()
+
+            subindent = indent + ' '*4
+            lines = _format_linedata(linedata, subindent, indent_width+4)
+            block = (indent
+                    + "Additional help topics (run `%s help TOPIC'):\n" % self.name
+                    + '\n'.join(lines)
+                    + "\n\n")
+        else:
+            block = ''
+        help = help.replace(indent+marker+suffix, block, 1)
+        return help
+
+    def _help_preprocess_cmd_name(self, help, cmdname=None):
+        marker = "${cmd_name}"
+        handler = self._get_cmd_handler(cmdname)
+        if not handler:
+            raise CmdlnError("cannot preprocess '%s' into help string: "
+                             "could not find command handler for %r" 
+                             % (marker, cmdname))
+        s = cmdname
+        if hasattr(handler, "aliases"):
+            s += " (%s)" % (", ".join(handler.aliases))
+        help = help.replace(marker, s)
+        return help
+
+    #TODO: this only makes sense as part of the Cmdln class.
+    #      Add hooks to add help preprocessing template vars and put
+    #      this one on that class.
+    def _help_preprocess_cmd_usage(self, help, cmdname=None):
+        marker = "${cmd_usage}"
+        handler = self._get_cmd_handler(cmdname)
+        if not handler:
+            raise CmdlnError("cannot preprocess '%s' into help string: "
+                             "could not find command handler for %r" 
+                             % (marker, cmdname))
+        indent, indent_width = _get_indent(marker, help)
+        suffix = _get_trailing_whitespace(marker, help)
+
+        # Extract the introspection bits we need.
+        func = handler.im_func
+        if func.func_defaults:
+            func_defaults = list(func.func_defaults)
+        else:
+            func_defaults = []
+        co_argcount = func.func_code.co_argcount
+        co_varnames = func.func_code.co_varnames
+        co_flags = func.func_code.co_flags
+        CO_FLAGS_ARGS = 4
+        CO_FLAGS_KWARGS = 8
+
+        # Adjust argcount for possible *args and **kwargs arguments.
+        argcount = co_argcount
+        if co_flags & CO_FLAGS_ARGS:   argcount += 1
+        if co_flags & CO_FLAGS_KWARGS: argcount += 1
+
+        # Determine the usage string.
+        usage = "%s %s" % (self.name, cmdname)
+        if argcount <= 2:   # handler ::= do_FOO(self, argv)
+            usage += " [ARGS...]"
+        elif argcount >= 3: # handler ::= do_FOO(self, subcmd, opts, ...)
+            argnames = list(co_varnames[3:argcount])
+            tail = ""
+            if co_flags & CO_FLAGS_KWARGS:
+                name = argnames.pop(-1)
+                import warnings
+                # There is no generally accepted mechanism for passing
+                # keyword arguments from the command line. Could
+                # *perhaps* consider: arg=value arg2=value2 ...
+                warnings.warn("argument '**%s' on '%s.%s' command "
+                              "handler will never get values" 
+                              % (name, self.__class__.__name__,
+                                 func.func_name))
+            if co_flags & CO_FLAGS_ARGS:
+                name = argnames.pop(-1)
+                tail = "[%s...]" % name.upper()
+            while func_defaults:
+                func_defaults.pop(-1)
+                name = argnames.pop(-1)
+                tail = "[%s%s%s]" % (name.upper(), (tail and ' ' or ''), tail)
+            while argnames:
+                name = argnames.pop(-1)
+                tail = "%s %s" % (name.upper(), tail)
+            usage += ' ' + tail
+
+        block_lines = [
+            self.helpindent + "Usage:",
+            self.helpindent + ' '*4 + usage
+        ]
+        block = '\n'.join(block_lines) + '\n\n'
+
+        help = help.replace(indent+marker+suffix, block, 1)
+        return help
+
+    #TODO: this only makes sense as part of the Cmdln class.
+    #      Add hooks to add help preprocessing template vars and put
+    #      this one on that class.
+    def _help_preprocess_cmd_option_list(self, help, cmdname=None):
+        marker = "${cmd_option_list}"
+        handler = self._get_cmd_handler(cmdname)
+        if not handler:
+            raise CmdlnError("cannot preprocess '%s' into help string: "
+                             "could not find command handler for %r" 
+                             % (marker, cmdname))
+        indent, indent_width = _get_indent(marker, help)
+        suffix = _get_trailing_whitespace(marker, help)
+        if hasattr(handler, "optparser"):
+            # Setup formatting options and format.
+            # - Indentation of 4 is better than optparse default of 2.
+            #   C.f. Damian Conway's discussion of this in Perl Best
+            #   Practices.
+            handler.optparser.formatter.indent_increment = 4
+            handler.optparser.formatter.current_indent = indent_width
+            block = handler.optparser.format_option_help() + '\n'
+        else:
+            block = ""
+
+        help = help.replace(indent+marker+suffix, block, 1)
+        return help
+
+    def _get_canonical_cmd_name(self, token):
+        map = self._get_canonical_map()
+        return map.get(token, None)
+
+    def _get_canonical_map(self):
+        """Return a mapping of available command names and aliases to
+        their canonical command name.
+        """
+        cacheattr = "_token2canonical"
+        if not hasattr(self, cacheattr):
+            # Get the list of commands and their aliases, if any.
+            token2canonical = {}
+            cmd2funcname = {} # use a dict to strip duplicates
+            for attr in self.get_names():
+                if attr.startswith("do_"):    cmdname = attr[3:]
+                elif attr.startswith("_do_"): cmdname = attr[4:]
+                else:
+                    continue
+                cmd2funcname[cmdname] = attr
+                token2canonical[cmdname] = cmdname
+            for cmdname, funcname in cmd2funcname.items(): # add aliases
+                func = getattr(self, funcname)
+                aliases = getattr(func, "aliases", [])
+                for alias in aliases:
+                    if alias in cmd2funcname:
+                        import warnings
+                        warnings.warn("'%s' alias for '%s' command conflicts "
+                                      "with '%s' handler"
+                                      % (alias, cmdname, cmd2funcname[alias]))
+                        continue
+                    token2canonical[alias] = cmdname
+            setattr(self, cacheattr, token2canonical)
+        return getattr(self, cacheattr)
+
+    def _get_cmd_handler(self, cmdname):
+        handler = None
+        try:
+            handler = getattr(self, 'do_' + cmdname)
+        except AttributeError:
+            try:
+                # Private command handlers begin with "_do_".
+                handler = getattr(self, '_do_' + cmdname)
+            except AttributeError:
+                pass
+        return handler
+
+    def _do_EOF(self, argv):
+        # Default EOF handler
+        # Note: an actual EOF is redirected to this command.
+        #TODO: separate name for this. Currently it is available from
+        #      command-line. Is that okay?
+        self.stdout.write('\n')
+        self.stdout.flush()
+        self.stop = True
+
+    def emptyline(self):
+        # Different from cmd.Cmd: don't repeat the last command for an
+        # emptyline.
+        if self.cmdlooping:
+            pass
+        else:
+            return self.do_help(["help"])
+
+
+#---- optparse.py extension to fix (IMO) some deficiencies
+#
+# See the class _OptionParserEx docstring for details.
+#
+
+class StopOptionProcessing(Exception):
+    """Indicate that option *and argument* processing should stop
+    cleanly. This is not an error condition. It is similar in spirit to
+    StopIteration. This is raised by _OptionParserEx's default "help"
+    and "version" option actions and can be raised by custom option
+    callbacks too.
+    
+    Hence the typical CmdlnOptionParser (a subclass of _OptionParserEx)
+    usage is:
+
+        parser = CmdlnOptionParser(mycmd)
+        parser.add_option("-f", "--force", dest="force")
+        ...
+        try:
+            opts, args = parser.parse_args()
+        except StopOptionProcessing:
+            # normal termination, "--help" was probably given
+            sys.exit(0)
+    """
+
+class _OptionParserEx(optparse.OptionParser):
+    """An optparse.OptionParser that uses exceptions instead of sys.exit.
+
+    This class is an extension of optparse.OptionParser that differs
+    as follows:
+    - Correct (IMO) the default OptionParser error handling to never
+      sys.exit(). Instead OptParseError exceptions are passed through.
+    - Add the StopOptionProcessing exception (a la StopIteration) to
+      indicate normal termination of option processing.
+      See StopOptionProcessing's docstring for details.
+
+    I'd also like to see the following in the core optparse.py, perhaps
+    as a RawOptionParser which would serve as a base class for the more
+    generally used OptionParser (that works as current):
+    - Remove the implicit addition of the -h|--help and --version
+      options. They can get in the way (e.g. if want '-?' and '-V' for
+      these as well) and it is not hard to do:
+        optparser.add_option("-h", "--help", action="help")
+        optparser.add_option("--version", action="version")
+      These are good practices, just not valid defaults if they can
+      get in the way.
+    """
+    def error(self, msg):
+        raise optparse.OptParseError(msg)
+
+    def exit(self, status=0, msg=None):
+        if status == 0:
+            raise StopOptionProcessing(msg)
+        else:
+            #TODO: don't lose status info here
+            raise optparse.OptParseError(msg)
+
+
+
+#---- optparse.py-based option processing support
+
+class CmdlnOptionParser(_OptionParserEx):
+    """An optparse.OptionParser class more appropriate for top-level
+    Cmdln options. For parsing of sub-command options, see
+    SubCmdOptionParser.
+
+    Changes:
+    - disable_interspersed_args() by default, because a Cmdln instance
+      has sub-commands which may themselves have options.
+    - Redirect print_help() to the Cmdln.do_help() which is better
+      equiped to handle the "help" action.
+    - error() will raise a CmdlnUserError: OptionParse.error() is meant
+      to be called for user errors. Raising a well-known error here can
+      make error handling clearer.
+    - Also see the changes in _OptionParserEx.
+    """
+    def __init__(self, cmdln, **kwargs):
+        self.cmdln = cmdln
+        kwargs["prog"] = self.cmdln.name
+        _OptionParserEx.__init__(self, **kwargs)
+        self.disable_interspersed_args()
+
+    def print_help(self, file=None):
+        self.cmdln.onecmd(["help"])
+
+    def error(self, msg):
+        raise CmdlnUserError(msg)
+
+
+class SubCmdOptionParser(_OptionParserEx):
+    def set_cmdln_info(self, cmdln, subcmd):
+        """Called by Cmdln to pass relevant info about itself needed
+        for print_help().
+        """
+        self.cmdln = cmdln
+        self.subcmd = subcmd
+
+    def print_help(self, file=None):
+        self.cmdln.onecmd(["help", self.subcmd])
+
+    def error(self, msg):
+        raise CmdlnUserError(msg)
+
+
+def option(*args, **kwargs):
+    """Decorator to add an option to the optparser argument of a Cmdln
+    subcommand.
+    
+    Example:
+        class MyShell(cmdln.Cmdln):
+            @cmdln.option("-f", "--force", help="force removal")
+            def do_remove(self, subcmd, opts, *args):
+                #...
+    """
+    #XXX Is there a possible optimization for many options to not have a
+    #    large stack depth here?
+    def decorate(f):
+        if not hasattr(f, "optparser"):
+            f.optparser = SubCmdOptionParser()
+        f.optparser.add_option(*args, **kwargs)
+        return f
+    return decorate
+
+
+class Cmdln(RawCmdln):
+    """An improved (on cmd.Cmd) framework for building multi-subcommand
+    scripts (think "svn" & "cvs") and simple shells (think "pdb" and
+    "gdb").
+
+    A simple example:
+
+        import cmdln
+
+        class MySVN(cmdln.Cmdln):
+            name = "svn"
+
+            @cmdln.aliases('stat', 'st')
+            @cmdln.option('-v', '--verbose', action='store_true'
+                          help='print verbose information')
+            def do_status(self, subcmd, opts, *paths):
+                print "handle 'svn status' command"
+
+            #...
+
+        if __name__ == "__main__":
+            shell = MySVN()
+            retval = shell.main()
+            sys.exit(retval)
+
+    'Cmdln' extends 'RawCmdln' by providing optparse option processing
+    integration.  See this class' _dispatch_cmd() docstring and
+    <http://trentm.com/projects/cmdln> for more information.
+    """
+    def _dispatch_cmd(self, handler, argv):
+        """Introspect sub-command handler signature to determine how to
+        dispatch the command. The raw handler provided by the base
+        'RawCmdln' class is still supported:
+
+            def do_foo(self, argv):
+                # 'argv' is the vector of command line args, argv[0] is
+                # the command name itself (i.e. "foo" or an alias)
+                pass
+
+        In addition, if the handler has more than 2 arguments option
+        processing is automatically done (using optparse):
+
+            @cmdln.option('-v', '--verbose', action='store_true')
+            def do_bar(self, subcmd, opts, *args):
+                # subcmd = <"bar" or an alias>
+                # opts = <an optparse.Values instance>
+                if opts.verbose:
+                    print "lots of debugging output..."
+                # args = <tuple of arguments>
+                for arg in args:
+                    bar(arg)
+
+        TODO: explain that "*args" can be other signatures as well.
+
+        The `cmdln.option` decorator corresponds to an `add_option()`
+        method call on an `optparse.OptionParser` instance.
+
+        You can declare a specific number of arguments:
+
+            @cmdln.option('-v', '--verbose', action='store_true')
+            def do_bar2(self, subcmd, opts, bar_one, bar_two):
+                #...
+
+        and an appropriate error message will be raised/printed if the
+        command is called with a different number of args.
+        """
+        co_argcount = handler.im_func.func_code.co_argcount
+        if co_argcount == 2:   # handler ::= do_foo(self, argv)
+            return handler(argv)
+        elif co_argcount >= 3: # handler ::= do_foo(self, subcmd, opts, ...)
+            try:
+                optparser = handler.optparser
+            except AttributeError:
+                optparser = handler.im_func.optparser = SubCmdOptionParser()
+            assert isinstance(optparser, SubCmdOptionParser)
+            optparser.set_cmdln_info(self, argv[0])
+            try:
+                opts, args = optparser.parse_args(argv[1:])
+            except StopOptionProcessing:
+                #TODO: this doesn't really fly for a replacement of
+                #      optparse.py behaviour, does it?
+                return 0 # Normal command termination
+
+            try:
+                return handler(argv[0], opts, *args)
+            except TypeError, ex:
+                # Some TypeError's are user errors:
+                #   do_foo() takes at least 4 arguments (3 given)
+                #   do_foo() takes at most 5 arguments (6 given)
+                #   do_foo() takes exactly 5 arguments (6 given)
+                # Raise CmdlnUserError for these with a suitably
+                # massaged error message.
+                import sys
+                tb = sys.exc_info()[2] # the traceback object
+                if tb.tb_next is not None:
+                    # If the traceback is more than one level deep, then the
+                    # TypeError do *not* happen on the "handler(...)" call
+                    # above. In that we don't want to handle it specially
+                    # here: it would falsely mask deeper code errors.
+                    raise
+                msg = ex.args[0]
+                match = _INCORRECT_NUM_ARGS_RE.search(msg)
+                if match:
+                    msg = list(match.groups())
+                    msg[1] = int(msg[1]) - 3
+                    if msg[1] == 1:
+                        msg[2] = msg[2].replace("arguments", "argument")
+                    msg[3] = int(msg[3]) - 3
+                    msg = ''.join(map(str, msg))
+                    raise CmdlnUserError(msg)
+                else:
+                    raise
+        else:
+            raise CmdlnError("incorrect argcount for %s(): takes %d, must "
+                             "take 2 for 'argv' signature or 3+ for 'opts' "
+                             "signature" % (handler.__name__, co_argcount))
+        
+
+
+#---- internal support functions
+
+def _format_linedata(linedata, indent, indent_width):
+    """Format specific linedata into a pleasant layout.
+    
+        "linedata" is a list of 2-tuples of the form:
+            (<item-display-string>, <item-docstring>)
+        "indent" is a string to use for one level of indentation
+        "indent_width" is a number of columns by which the
+            formatted data will be indented when printed.
+
+    The <item-display-string> column is held to 15 columns.
+    """
+    lines = []
+    WIDTH = 78 - indent_width
+    SPACING = 2
+    NAME_WIDTH_LOWER_BOUND = 13
+    NAME_WIDTH_UPPER_BOUND = 16
+    NAME_WIDTH = max([len(s) for s,d in linedata])
+    if NAME_WIDTH < NAME_WIDTH_LOWER_BOUND:
+        NAME_WIDTH = NAME_WIDTH_LOWER_BOUND
+    else:
+        NAME_WIDTH = NAME_WIDTH_UPPER_BOUND
+
+    DOC_WIDTH = WIDTH - NAME_WIDTH - SPACING
+    for namestr, doc in linedata:
+        line = indent + namestr
+        if len(namestr) <= NAME_WIDTH:
+            line += ' ' * (NAME_WIDTH + SPACING - len(namestr))
+        else:
+            lines.append(line)
+            line = indent + ' ' * (NAME_WIDTH + SPACING)
+        line += _summarize_doc(doc, DOC_WIDTH)
+        lines.append(line.rstrip())
+    return lines
+
+def _summarize_doc(doc, length=60):
+    r"""Parse out a short one line summary from the given doclines.
+    
+        "doc" is the doc string to summarize.
+        "length" is the max length for the summary
+
+    >>> _summarize_doc("this function does this")
+    'this function does this'
+    >>> _summarize_doc("this function does this", 10)
+    'this fu...'
+    >>> _summarize_doc("this function does this\nand that")
+    'this function does this and that'
+    >>> _summarize_doc("this function does this\n\nand that")
+    'this function does this'
+    """
+    import re
+    if doc is None:
+        return ""
+    assert length > 3, "length <= 3 is absurdly short for a doc summary"
+    doclines = doc.strip().splitlines(0)
+    if not doclines:
+        return ""
+
+    summlines = []
+    for i, line in enumerate(doclines):
+        stripped = line.strip()
+        if not stripped:
+            break
+        summlines.append(stripped)
+        if len(''.join(summlines)) >= length:
+            break
+
+    summary = ' '.join(summlines)
+    if len(summary) > length:
+        summary = summary[:length-3] + "..." 
+    return summary
+
+
+def line2argv(line):
+    r"""Parse the given line into an argument vector.
+    
+        "line" is the line of input to parse.
+
+    This may get niggly when dealing with quoting and escaping. The
+    current state of this parsing may not be completely thorough/correct
+    in this respect.
+    
+    >>> from cmdln import line2argv
+    >>> line2argv("foo")
+    ['foo']
+    >>> line2argv("foo bar")
+    ['foo', 'bar']
+    >>> line2argv("foo bar ")
+    ['foo', 'bar']
+    >>> line2argv(" foo bar")
+    ['foo', 'bar']
+
+    Quote handling:
+    
+    >>> line2argv("'foo bar'")
+    ['foo bar']
+    >>> line2argv('"foo bar"')
+    ['foo bar']
+    >>> line2argv(r'"foo\"bar"')
+    ['foo"bar']
+    >>> line2argv("'foo bar' spam")
+    ['foo bar', 'spam']
+    >>> line2argv("'foo 'bar spam")
+    ['foo bar', 'spam']
+    
+    >>> line2argv('some\tsimple\ttests')
+    ['some', 'simple', 'tests']
+    >>> line2argv('a "more complex" test')
+    ['a', 'more complex', 'test']
+    >>> line2argv('a more="complex test of " quotes')
+    ['a', 'more=complex test of ', 'quotes']
+    >>> line2argv('a more" complex test of " quotes')
+    ['a', 'more complex test of ', 'quotes']
+    >>> line2argv('an "embedded \\"quote\\""')
+    ['an', 'embedded "quote"']
+
+    # Komodo bug 48027
+    >>> line2argv('foo bar C:\\')
+    ['foo', 'bar', 'C:\\']
+
+    # Komodo change 127581
+    >>> line2argv(r'"\test\slash" "foo bar" "foo\"bar"')
+    ['\\test\\slash', 'foo bar', 'foo"bar']
+
+    # Komodo change 127629
+    >>> if sys.platform == "win32":
+    ...     line2argv(r'\foo\bar') == ['\\foo\\bar']
+    ...     line2argv(r'\\foo\\bar') == ['\\\\foo\\\\bar']
+    ...     line2argv('"foo') == ['foo']
+    ... else:
+    ...     line2argv(r'\foo\bar') == ['foobar']
+    ...     line2argv(r'\\foo\\bar') == ['\\foo\\bar']
+    ...     try:
+    ...         line2argv('"foo')
+    ...     except ValueError, ex:
+    ...         "not terminated" in str(ex)
+    True
+    True
+    True
+    """
+    import string
+    line = line.strip()
+    argv = []
+    state = "default"
+    arg = None  # the current argument being parsed
+    i = -1
+    while 1:
+        i += 1
+        if i >= len(line): break
+        ch = line[i]
+
+        if ch == "\\" and i+1 < len(line):
+            # escaped char always added to arg, regardless of state
+            if arg is None: arg = ""
+            if (sys.platform == "win32"
+                or state in ("double-quoted", "single-quoted")
+               ) and line[i+1] not in tuple('"\''):
+                arg += ch
+            i += 1
+            arg += line[i]
+            continue
+
+        if state == "single-quoted":
+            if ch == "'":
+                state = "default"
+            else:
+                arg += ch
+        elif state == "double-quoted":
+            if ch == '"':
+                state = "default"
+            else:
+                arg += ch
+        elif state == "default":
+            if ch == '"':
+                if arg is None: arg = ""
+                state = "double-quoted"
+            elif ch == "'":
+                if arg is None: arg = ""
+                state = "single-quoted"
+            elif ch in string.whitespace:
+                if arg is not None:
+                    argv.append(arg)
+                arg = None
+            else:
+                if arg is None: arg = ""
+                arg += ch
+    if arg is not None:
+        argv.append(arg)
+    if not sys.platform == "win32" and state != "default":
+        raise ValueError("command line is not terminated: unfinished %s "
+                         "segment" % state)
+    return argv
+
+
+def argv2line(argv):
+    r"""Put together the given argument vector into a command line.
+    
+        "argv" is the argument vector to process.
+    
+    >>> from cmdln import argv2line
+    >>> argv2line(['foo'])
+    'foo'
+    >>> argv2line(['foo', 'bar'])
+    'foo bar'
+    >>> argv2line(['foo', 'bar baz'])
+    'foo "bar baz"'
+    >>> argv2line(['foo"bar'])
+    'foo"bar'
+    >>> print argv2line(['foo" bar'])
+    'foo" bar'
+    >>> print argv2line(["foo' bar"])
+    "foo' bar"
+    >>> argv2line(["foo'bar"])
+    "foo'bar"
+    """
+    escapedArgs = []
+    for arg in argv:
+        if ' ' in arg and '"' not in arg:
+            arg = '"'+arg+'"'
+        elif ' ' in arg and "'" not in arg:
+            arg = "'"+arg+"'"
+        elif ' ' in arg:
+            arg = arg.replace('"', r'\"')
+            arg = '"'+arg+'"'
+        escapedArgs.append(arg)
+    return ' '.join(escapedArgs)
+
+
+# Recipe: dedent (0.1) in /Users/trentm/tm/recipes/cookbook
+def _dedentlines(lines, tabsize=8, skip_first_line=False):
+    """_dedentlines(lines, tabsize=8, skip_first_line=False) -> dedented lines
+    
+        "lines" is a list of lines to dedent.
+        "tabsize" is the tab width to use for indent width calculations.
+        "skip_first_line" is a boolean indicating if the first line should
+            be skipped for calculating the indent width and for dedenting.
+            This is sometimes useful for docstrings and similar.
+    
+    Same as dedent() except operates on a sequence of lines. Note: the
+    lines list is modified **in-place**.
+    """
+    DEBUG = False
+    if DEBUG: 
+        print "dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\
+              % (tabsize, skip_first_line)
+    indents = []
+    margin = None
+    for i, line in enumerate(lines):
+        if i == 0 and skip_first_line: continue
+        indent = 0
+        for ch in line:
+            if ch == ' ':
+                indent += 1
+            elif ch == '\t':
+                indent += tabsize - (indent % tabsize)
+            elif ch in '\r\n':
+                continue # skip all-whitespace lines
+            else:
+                break
+        else:
+            continue # skip all-whitespace lines
+        if DEBUG: print "dedent: indent=%d: %r" % (indent, line)
+        if margin is None:
+            margin = indent
+        else:
+            margin = min(margin, indent)
+    if DEBUG: print "dedent: margin=%r" % margin
+
+    if margin is not None and margin > 0:
+        for i, line in enumerate(lines):
+            if i == 0 and skip_first_line: continue
+            removed = 0
+            for j, ch in enumerate(line):
+                if ch == ' ':
+                    removed += 1
+                elif ch == '\t':
+                    removed += tabsize - (removed % tabsize)
+                elif ch in '\r\n':
+                    if DEBUG: print "dedent: %r: EOL -> strip up to EOL" % line
+                    lines[i] = lines[i][j:]
+                    break
+                else:
+                    raise ValueError("unexpected non-whitespace char %r in "
+                                     "line %r while removing %d-space margin"
+                                     % (ch, line, margin))
+                if DEBUG:
+                    print "dedent: %r: %r -> removed %d/%d"\
+                          % (line, ch, removed, margin)
+                if removed == margin:
+                    lines[i] = lines[i][j+1:]
+                    break
+                elif removed > margin:
+                    lines[i] = ' '*(removed-margin) + lines[i][j+1:]
+                    break
+    return lines
+
+def _dedent(text, tabsize=8, skip_first_line=False):
+    """_dedent(text, tabsize=8, skip_first_line=False) -> dedented text
+
+        "text" is the text to dedent.
+        "tabsize" is the tab width to use for indent width calculations.
+        "skip_first_line" is a boolean indicating if the first line should
+            be skipped for calculating the indent width and for dedenting.
+            This is sometimes useful for docstrings and similar.
+    
+    textwrap.dedent(s), but don't expand tabs to spaces
+    """
+    lines = text.splitlines(1)
+    _dedentlines(lines, tabsize=tabsize, skip_first_line=skip_first_line)
+    return ''.join(lines)
+
+
+def _get_indent(marker, s, tab_width=8):
+    """_get_indent(marker, s, tab_width=8) ->
+        (<indentation-of-'marker'>, <indentation-width>)"""
+    # Figure out how much the marker is indented.
+    INDENT_CHARS = tuple(' \t')
+    start = s.index(marker)
+    i = start
+    while i > 0:
+        if s[i-1] not in INDENT_CHARS:
+            break
+        i -= 1
+    indent = s[i:start]
+    indent_width = 0
+    for ch in indent:
+        if ch == ' ':
+            indent_width += 1
+        elif ch == '\t':
+            indent_width += tab_width - (indent_width % tab_width)
+    return indent, indent_width
+
+def _get_trailing_whitespace(marker, s):
+    """Return the whitespace content trailing the given 'marker' in string 's',
+    up to and including a newline.
+    """
+    suffix = ''
+    start = s.index(marker) + len(marker)
+    i = start
+    while i < len(s):
+        if s[i] in ' \t':
+            suffix += s[i]
+        elif s[i] in '\r\n':
+            suffix += s[i]
+            if s[i] == '\r' and i+1 < len(s) and s[i+1] == '\n':
+                suffix += s[i+1]
+            break
+        else:
+            break
+        i += 1
+    return suffix
+
+
+
+#---- bash completion support
+# Note: This is still experimental. I expect to change this
+# significantly.
+#
+# To get Bash completion for a cmdln.Cmdln class, run the following
+# bash command:
+#   $ complete -C 'python -m cmdln /path/to/script.py CmdlnClass' cmdname
+# For example:
+#   $ complete -C 'python -m cmdln ~/bin/svn.py SVN' svn
+#
+#TODO: Simplify the above so don't have to given path to script (try to
+#      find it on PATH, if possible). Could also make class name
+#      optional if there is only one in the module (common case).
+
+if __name__ == "__main__" and len(sys.argv) == 6:
+    def _log(s):
+        return # no-op, comment out for debugging
+        from os.path import expanduser
+        fout = open(expanduser("~/tmp/bashcpln.log"), 'a')
+        fout.write(str(s) + '\n')
+        fout.close()
+
+    # Recipe: module_from_path (1.0.1+)
+    def _module_from_path(path):
+        import imp, os, sys
+        path = os.path.expanduser(path)
+        dir = os.path.dirname(path) or os.curdir
+        name = os.path.splitext(os.path.basename(path))[0]
+        sys.path.insert(0, dir)
+        try:
+            iinfo = imp.find_module(name, [dir])
+            return imp.load_module(name, *iinfo)
+        finally:
+            sys.path.remove(dir)
+
+    def _get_bash_cplns(script_path, class_name, cmd_name,
+                        token, preceding_token):
+        _log('--')
+        _log('get_cplns(%r, %r, %r, %r, %r)'
+             % (script_path, class_name, cmd_name, token, preceding_token))
+        comp_line = os.environ["COMP_LINE"]
+        comp_point = int(os.environ["COMP_POINT"])
+        _log("COMP_LINE: %r" % comp_line)
+        _log("COMP_POINT: %r" % comp_point)
+
+        try:
+            script = _module_from_path(script_path)
+        except ImportError, ex:
+            _log("error importing `%s': %s" % (script_path, ex))
+            return []
+        shell = getattr(script, class_name)()
+        cmd_map = shell._get_canonical_map()
+        del cmd_map["EOF"]
+
+        # Determine if completing the sub-command name.
+        parts = comp_line[:comp_point].split(None, 1)
+        _log(parts)
+        if len(parts) == 1 or not (' ' in parts[1] or '\t' in parts[1]):
+            #TODO: if parts[1].startswith('-'): handle top-level opts
+            _log("complete sub-command names")
+            matches = {}
+            for name, canon_name in cmd_map.items():
+                if name.startswith(token):
+                    matches[name] = canon_name
+            if not matches:
+                return []
+            elif len(matches) == 1:
+                return matches.keys()
+            elif len(set(matches.values())) == 1:
+                return [matches.values()[0]]
+            else:
+                return matches.keys()
+
+        # Otherwise, complete options for the given sub-command.
+        #TODO: refine this so it does the right thing with option args
+        if token.startswith('-'):
+            cmd_name = comp_line.split(None, 2)[1]
+            try:
+                cmd_canon_name = cmd_map[cmd_name]
+            except KeyError:
+                return []
+            handler = shell._get_cmd_handler(cmd_canon_name)
+            optparser = getattr(handler, "optparser", None)
+            if optparser is None:
+                optparser = SubCmdOptionParser()
+            opt_strs = []
+            for option in optparser.option_list:
+                for opt_str in option._short_opts + option._long_opts:
+                    if opt_str.startswith(token):
+                        opt_strs.append(opt_str)
+            return opt_strs
+
+        return []
+
+    for cpln in _get_bash_cplns(*sys.argv[1:]):
+        print cpln
+
diff --git a/scripts/lib/wic/utils/errors.py b/scripts/lib/wic/utils/errors.py
new file mode 100644
index 0000000000..9410311875
--- /dev/null
+++ b/scripts/lib/wic/utils/errors.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python -tt
+#
+# Copyright (c) 2007 Red Hat, Inc.
+# Copyright (c) 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+class CreatorError(Exception):
+    """An exception base class for all imgcreate errors."""
+    keyword = '<creator>'
+
+    def __init__(self, msg):
+        self.msg = msg
+
+    def __str__(self):
+        if isinstance(self.msg, unicode):
+            self.msg = self.msg.encode('utf-8', 'ignore')
+        else:
+            self.msg = str(self.msg)
+        return self.keyword + self.msg
+
+class Usage(CreatorError):
+    keyword = '<usage>'
+
+    def __str__(self):
+        if isinstance(self.msg, unicode):
+            self.msg = self.msg.encode('utf-8', 'ignore')
+        else:
+            self.msg = str(self.msg)
+        return self.keyword + self.msg + ', please use "--help" for more info'
+
+class KsError(CreatorError):
+    keyword = '<kickstart>'
+
+class ImageError(CreatorError):
+    keyword = '<mount>'
diff --git a/scripts/lib/wic/utils/fs_related.py b/scripts/lib/wic/utils/fs_related.py
new file mode 100644
index 0000000000..ea9f85c60f
--- /dev/null
+++ b/scripts/lib/wic/utils/fs_related.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python -tt
+#
+# Copyright (c) 2007, Red Hat, Inc.
+# Copyright (c) 2009, 2010, 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+from __future__ import with_statement
+import os
+import sys
+import errno
+import stat
+import random
+import string
+import time
+import uuid
+
+from wic import msger
+from wic.utils import runner
+from wic.utils.errors import *
+from wic.utils.oe.misc import *
+
+def find_binary_path(binary):
+    if os.environ.has_key("PATH"):
+        paths = os.environ["PATH"].split(":")
+    else:
+        paths = []
+        if os.environ.has_key("HOME"):
+            paths += [os.environ["HOME"] + "/bin"]
+        paths += ["/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"]
+
+    for path in paths:
+        bin_path = "%s/%s" % (path, binary)
+        if os.path.exists(bin_path):
+            return bin_path
+
+    print "External command '%s' not found, exiting." % binary
+    print "  (Please install '%s' on your host system)" % binary
+    sys.exit(1)
+
+def makedirs(dirname):
+    """A version of os.makedirs() that doesn't throw an
+    exception if the leaf directory already exists.
+    """
+    try:
+        os.makedirs(dirname)
+    except OSError, err:
+        if err.errno != errno.EEXIST:
+            raise
+
+class Disk:
+    """
+    Generic base object for a disk.
+    """
+    def __init__(self, size, device = None):
+        self._device = device
+        self._size = size
+
+    def create(self):
+        pass
+
+    def cleanup(self):
+        pass
+
+    def get_device(self):
+        return self._device
+    def set_device(self, path):
+        self._device = path
+    device = property(get_device, set_device)
+
+    def get_size(self):
+        return self._size
+    size = property(get_size)
+
+
+class DiskImage(Disk):
+    """
+    A Disk backed by a file.
+    """
+    def __init__(self, image_file, size):
+        Disk.__init__(self, size)
+        self.image_file = image_file
+
+    def exists(self):
+        return os.path.exists(self.image_file)
+
+    def create(self):
+        if self.device is not None:
+            return
+
+        blocks = self.size / 1024
+        if self.size - blocks * 1024:
+            blocks += 1
+
+        # create disk image
+        dd_cmd = "dd if=/dev/zero of=%s bs=1024 seek=%d count=1" % \
+            (self.image_file, blocks)
+        exec_cmd(dd_cmd)
+
+        self.device = self.image_file
diff --git a/scripts/lib/wic/utils/misc.py b/scripts/lib/wic/utils/misc.py
new file mode 100644
index 0000000000..6e56316608
--- /dev/null
+++ b/scripts/lib/wic/utils/misc.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python -tt
+#
+# Copyright (c) 2010, 2011 Intel Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os
+import sys
+import time
+
+def build_name(kscfg, release=None, prefix = None, suffix = None):
+    """Construct and return an image name string.
+
+    This is a utility function to help create sensible name and fslabel
+    strings. The name is constructed using the sans-prefix-and-extension
+    kickstart filename and the supplied prefix and suffix.
+
+    kscfg -- a path to a kickstart file
+    release --  a replacement to suffix for image release
+    prefix -- a prefix to prepend to the name; defaults to None, which causes
+              no prefix to be used
+    suffix -- a suffix to append to the name; defaults to None, which causes
+              a YYYYMMDDHHMM suffix to be used
+
+    Note, if maxlen is less then the len(suffix), you get to keep both pieces.
+
+    """
+    name = os.path.basename(kscfg)
+    idx = name.rfind('.')
+    if idx >= 0:
+        name = name[:idx]
+
+    if release is not None:
+        suffix = ""
+    if prefix is None:
+        prefix = ""
+    if suffix is None:
+        suffix = time.strftime("%Y%m%d%H%M")
+
+    if name.startswith(prefix):
+        name = name[len(prefix):]
+
+    prefix = "%s-" % prefix if prefix else ""
+    suffix = "-%s" % suffix if suffix else ""
+
+    ret = prefix + name + suffix
+
+    return ret
diff --git a/scripts/lib/wic/utils/oe/__init__.py b/scripts/lib/wic/utils/oe/__init__.py
new file mode 100644
index 0000000000..0a81575a74
--- /dev/null
+++ b/scripts/lib/wic/utils/oe/__init__.py
@@ -0,0 +1,22 @@
+#
+# OpenEmbedded wic utils library
+#
+# Copyright (c) 2013, Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] linux.intel.com>
+#
diff --git a/scripts/lib/wic/utils/oe/misc.py b/scripts/lib/wic/utils/oe/misc.py
new file mode 100644
index 0000000000..b0b5baab73
--- /dev/null
+++ b/scripts/lib/wic/utils/oe/misc.py
@@ -0,0 +1,205 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (c) 2013, Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION
+# This module provides a place to collect various wic-related utils
+# for the OpenEmbedded Image Tools.
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] linux.intel.com>
+#
+
+from wic import msger
+from wic.utils import runner
+
+def __exec_cmd(cmd_and_args, as_shell = False, catch = 3):
+    """
+    Execute command, catching stderr, stdout
+
+    Need to execute as_shell if the command uses wildcards
+    """
+    msger.debug("__exec_cmd: %s" % cmd_and_args)
+    args = cmd_and_args.split()
+    msger.debug(args)
+
+    if (as_shell):
+        rc, out = runner.runtool(cmd_and_args, catch)
+    else:
+        rc, out = runner.runtool(args, catch)
+    out = out.strip()
+    msger.debug("__exec_cmd: output for %s (rc = %d): %s" % \
+                (cmd_and_args, rc, out))
+
+    return (rc, out)
+
+
+def exec_cmd(cmd_and_args, as_shell = False, catch = 3):
+    """
+    Execute command, catching stderr, stdout
+
+    Exits if rc non-zero
+    """
+    rc, out = __exec_cmd(cmd_and_args, as_shell, catch)
+
+    if rc != 0:
+        msger.error("exec_cmd: %s returned '%s' instead of 0" % (cmd_and_args, rc))
+
+    return out
+
+
+def exec_cmd_quiet(cmd_and_args, as_shell = False):
+    """
+    Execute command, catching nothing in the output
+
+    Exits if rc non-zero
+    """
+    return exec_cmd(cmd_and_args, as_shell, 0)
+
+
+def exec_native_cmd(cmd_and_args, native_sysroot, catch = 3):
+    """
+    Execute native command, catching stderr, stdout
+
+    Need to execute as_shell if the command uses wildcards
+
+    Always need to execute native commands as_shell
+    """
+    native_paths = \
+        "export PATH=%s/sbin:%s/usr/sbin:%s/usr/bin:$PATH" % \
+        (native_sysroot, native_sysroot, native_sysroot)
+    native_cmd_and_args = "%s;%s" % (native_paths, cmd_and_args)
+    msger.debug("exec_native_cmd: %s" % cmd_and_args)
+
+    args = cmd_and_args.split()
+    msger.debug(args)
+
+    rc, out = __exec_cmd(native_cmd_and_args, True, catch)
+
+    if rc == 127: # shell command-not-found
+        msger.error("A native (host) program required to build the image "
+                    "was not found (see details above). Please make sure "
+                    "it's installed and try again.")
+
+    return (rc, out)
+
+
+def exec_native_cmd_quiet(cmd_and_args, native_sysroot):
+    """
+    Execute native command, catching nothing in the output
+
+    Need to execute as_shell if the command uses wildcards
+
+    Always need to execute native commands as_shell
+    """
+    return exec_native_cmd(cmd_and_args, native_sysroot, 0)
+
+
+# kickstart doesn't support variable substution in commands, so this
+# is our current simplistic scheme for supporting that
+
+wks_vars = dict()
+
+def get_wks_var(key):
+    return wks_vars[key]
+
+def add_wks_var(key, val):
+    wks_vars[key] = val
+
+BOOTDD_EXTRA_SPACE = 16384
+IMAGE_EXTRA_SPACE = 10240
+IMAGE_OVERHEAD_FACTOR = 1.3
+
+__bitbake_env_lines = ""
+
+def set_bitbake_env_lines(bitbake_env_lines):
+    global __bitbake_env_lines
+    __bitbake_env_lines = bitbake_env_lines
+
+def get_bitbake_env_lines():
+    return __bitbake_env_lines
+
+def find_bitbake_env_lines(image_name):
+    """
+    If image_name is empty, plugins might still be able to use the
+    environment, so set it regardless.
+    """
+    if image_name:
+        bitbake_env_cmd = "bitbake -e %s" % image_name
+    else:
+        bitbake_env_cmd = "bitbake -e"
+    rc, bitbake_env_lines = __exec_cmd(bitbake_env_cmd)
+    if rc != 0:
+        print "Couldn't get '%s' output." % bitbake_env_cmd
+        return None
+
+    return bitbake_env_lines
+
+def find_artifact(bitbake_env_lines, variable):
+    """
+    Gather the build artifact for the current image (the image_name
+    e.g. core-image-minimal) for the current MACHINE set in local.conf
+    """
+    retval = ""
+
+    for line in bitbake_env_lines.split('\n'):
+        if (get_line_val(line, variable)):
+            retval = get_line_val(line, variable)
+            break
+
+    return retval
+
+def get_line_val(line, key):
+    """
+    Extract the value from the VAR="val" string
+    """
+    if line.startswith(key + "="):
+        stripped_line = line.split('=')[1]
+        stripped_line = stripped_line.replace('\"', '')
+        return stripped_line
+    return None
+
+def get_bitbake_var(key):
+    for line in __bitbake_env_lines.split('\n'):
+        if (get_line_val(line, key)):
+            val = get_line_val(line, key)
+            return val
+    return None
+
+def parse_sourceparams(sourceparams):
+    """
+    Split sourceparams string of the form key1=val1[,key2=val2,...]
+    into a dict.  Also accepts valueless keys i.e. without =.
+
+    Returns dict of param key/val pairs (note that val may be None).
+    """
+    params_dict = {}
+
+    params = sourceparams.split(',')
+    if params:
+        for p in params:
+            if not p:
+                continue
+            if not '=' in p:
+                key = p
+                val = None
+            else:
+                key, val = p.split('=')
+            params_dict[key] = val
+
+    return params_dict
diff --git a/scripts/lib/wic/utils/oe/package_manager.py b/scripts/lib/wic/utils/oe/package_manager.py
new file mode 100644
index 0000000000..92ce98e2ce
--- /dev/null
+++ b/scripts/lib/wic/utils/oe/package_manager.py
@@ -0,0 +1,810 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (c) 2014, Enea AB.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION
+# This implements the opkg package manager wrapper as a combination of
+# meta/lib/oe/package_manager.py and bitbake/lib/bb/utils.py files and
+# adaptation of those files to 'wic'.
+#
+# AUTHORS
+# Adrian Calianu <adrian.calianu (at] enea.com>
+#
+# This file incorporates work covered by the following copyright and
+# permission notice:
+#
+#     meta/COPYING.GPLv2 (GPLv2)
+#     meta/COPYING.MIT (MIT)
+#
+#     Copyright (C) 2004 Michael Lauer
+#
+#     Permission to use, copy, modify, and/or distribute this software
+#     for any purpose with or without fee is hereby granted, provided
+#     that the above copyright notice and this permission notice appear
+#     in all copies.
+#
+#     THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
+#     WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+#     WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
+#     AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR
+#     CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+#     OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+#     NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+#     CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
+from abc import ABCMeta, abstractmethod
+import os
+import glob
+import subprocess
+import shutil
+import multiprocessing
+import re
+import errno
+import fcntl
+
+from mic.utils.oe.misc import *
+from mic import msger
+
+def mkdirhier(directory):
+    """Create a directory like 'mkdir -p', but does not complain if
+    directory already exists like os.makedirs
+    """
+
+    try:
+        os.makedirs(directory)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise e
+
+def remove(path, recurse=False):
+    """Equivalent to rm -f or rm -rf"""
+    if not path:
+        return
+    if recurse:
+        # shutil.rmtree(name) would be ideal but its too slow
+        subprocess.call(['rm', '-rf'] + glob.glob(path))
+        return
+    for name in glob.glob(path):
+        try:
+            os.unlink(name)
+        except OSError as exc:
+            if exc.errno != errno.ENOENT:
+                raise
+
+def lockfile(name, shared=False, retry=True):
+    """
+    Use the file fn as a lock file, return when the lock has been acquired.
+    Returns a variable to pass to unlockfile().
+    """
+    dirname = os.path.dirname(name)
+    mkdirhier(dirname)
+
+    if not os.access(dirname, os.W_OK):
+        logger.error("Unable to acquire lock '%s', directory is not writable",
+                     name)
+        sys.exit(1)
+
+    op = fcntl.LOCK_EX
+    if shared:
+        op = fcntl.LOCK_SH
+    if not retry:
+        op = op | fcntl.LOCK_NB
+
+    while True:
+        # If we leave the lockfiles lying around there is no problem
+        # but we should clean up after ourselves. This gives potential
+        # for races though. To work around this, when we acquire the lock
+        # we check the file we locked was still the lock file on disk.
+        # by comparing inode numbers. If they don't match or the lockfile
+        # no longer exists, we start again.
+
+        # This implementation is unfair since the last person to request the
+        # lock is the most likely to win it.
+
+        try:
+            lf = open(name, 'a+')
+            fileno = lf.fileno()
+            fcntl.flock(fileno, op)
+            statinfo = os.fstat(fileno)
+            if os.path.exists(lf.name):
+                statinfo2 = os.stat(lf.name)
+                if statinfo.st_ino == statinfo2.st_ino:
+                    return lf
+            lf.close()
+        except Exception:
+            try:
+                lf.close()
+            except Exception:
+                pass
+            pass
+        if not retry:
+            return None
+
+def unlockfile(lf):
+    """
+    Unlock a file locked using lockfile()
+    """
+    try:
+        # If we had a shared lock, we need to promote to exclusive before
+        # removing the lockfile. Attempt this, ignore failures.
+        fcntl.flock(lf.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
+        os.unlink(lf.name)
+    except (IOError, OSError):
+        pass
+    fcntl.flock(lf.fileno(), fcntl.LOCK_UN)
+    lf.close()
+
+def which(path, item, direction = 0, history = False):
+    """
+    Locate a file in a PATH
+    """
+
+    hist = []
+    paths = (path or "").split(':')
+    if direction != 0:
+        paths.reverse()
+
+    for p in paths:
+        next = os.path.join(p, item)
+        hist.append(next)
+        if os.path.exists(next):
+            if not os.path.isabs(next):
+                next = os.path.abspath(next)
+            if history:
+                return next, hist
+            return next
+
+    if history:
+        return "", hist
+    return ""
+
+
+
+# this can be used by all PM backends to create the index files in parallel
+def wic_create_index(arg):
+    index_cmd = arg
+
+    try:
+        msger.info("Executing '%s' ..." % index_cmd)
+        subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True)
+    except subprocess.CalledProcessError as e:
+        return("Index creation command '%s' failed with return code %d:\n%s" %
+               (e.cmd, e.returncode, e.output))
+
+    return None
+
+
+class WicIndexer(object):
+    __metaclass__ = ABCMeta
+
+    def __init__(self, d, deploy_dir):
+        self.d = d
+        self.deploy_dir = deploy_dir
+
+    @abstractmethod
+    def write_index(self):
+        pass
+
+class WicOpkgIndexer(WicIndexer):
+    def write_index(self):
+        arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS",
+                     "SDK_PACKAGE_ARCHS",
+                     "MULTILIB_ARCHS"]
+
+        opkg_index_cmd = which(os.getenv('PATH'), "opkg-make-index")
+
+        if not os.path.exists(os.path.join(self.deploy_dir, "Packages")):
+            open(os.path.join(self.deploy_dir, "Packages"), "w").close()
+
+        index_cmds = []
+        for arch_var in arch_vars:
+            if self.d.has_key(arch_var):
+                archs = self.d[arch_var]
+            else:
+                archs = None
+
+            if archs is None:
+                continue
+
+            for arch in archs.split():
+                pkgs_dir = os.path.join(self.deploy_dir, arch)
+                pkgs_file = os.path.join(pkgs_dir, "Packages")
+
+                if not os.path.isdir(pkgs_dir):
+                    continue
+
+                if not os.path.exists(pkgs_file):
+                    open(pkgs_file, "w").close()
+
+                index_cmds.append('%s -r %s -p %s -m %s' %
+                                  (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir))
+
+        if len(index_cmds) == 0:
+            msger.info("There are no packages in %s!" % self.deploy_dir)
+            return
+
+        nproc = multiprocessing.cpu_count()
+        pool = multiprocessing.Pool(nproc)
+        results = list(pool.imap(wic_create_index, index_cmds))
+        pool.close()
+        pool.join()
+
+        for result in results:
+            if result is not None:
+                return(result)
+
+class WicPkgsList(object):
+    __metaclass__ = ABCMeta
+
+    def __init__(self, d, rootfs_dir):
+        self.d = d
+        self.rootfs_dir = rootfs_dir
+
+    @abstractmethod
+    def list(self, format=None):
+        pass
+
+
+class WicOpkgPkgsList(WicPkgsList):
+    def __init__(self, d, rootfs_dir, config_file):
+        super(WicOpkgPkgsList, self).__init__(d, rootfs_dir)
+
+        self.opkg_cmd = which(os.getenv('PATH'), "opkg-cl")
+        self.opkg_args = "-f %s -o %s " % (config_file, rootfs_dir)
+        if self.d.has_key("OPKG_ARGS"):
+            self.opkg_args += self.d["OPKG_ARGS"]
+
+    def list(self, format=None):
+        opkg_query_cmd = which(os.getenv('PATH'), "opkg-query-helper.py")
+
+        if format == "arch":
+            cmd = "%s %s status | %s -a" % \
+                (self.opkg_cmd, self.opkg_args, opkg_query_cmd)
+        elif format == "file":
+            cmd = "%s %s status | %s -f" % \
+                (self.opkg_cmd, self.opkg_args, opkg_query_cmd)
+        elif format == "ver":
+            cmd = "%s %s status | %s -v" % \
+                (self.opkg_cmd, self.opkg_args, opkg_query_cmd)
+        elif format == "deps":
+            cmd = "%s %s status | %s" % \
+                (self.opkg_cmd, self.opkg_args, opkg_query_cmd)
+        else:
+            cmd = "%s %s list_installed | cut -d' ' -f1" % \
+                (self.opkg_cmd, self.opkg_args)
+
+        try:
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip()
+        except subprocess.CalledProcessError as e:
+            msger.error("Cannot get the installed packages list. Command '%s' "
+                     "returned %d:\n%s" % (cmd, e.returncode, e.output))
+
+        if output and format == "file":
+            tmp_output = ""
+            for line in output.split('\n'):
+                pkg, pkg_file, pkg_arch = line.split()
+                full_path = os.path.join(self.rootfs_dir, pkg_arch, pkg_file)
+                if os.path.exists(full_path):
+                    tmp_output += "%s %s %s\n" % (pkg, full_path, pkg_arch)
+                else:
+                    tmp_output += "%s %s %s\n" % (pkg, pkg_file, pkg_arch)
+
+            output = tmp_output
+
+        return output
+
+
+class WicPackageManager(object):
+    """
+    This is an abstract class. Do not instantiate this directly.
+    """
+    __metaclass__ = ABCMeta
+
+    def __init__(self, d, pseudo, native_sysroot):
+        self.d = d
+        self.deploy_dir = None
+        self.deploy_lock = None
+        if self.d.has_key('PACKAGE_FEED_URIS'):
+            self.feed_uris = self.d['PACKAGE_FEED_URIS']
+        else:
+            self.feed_uris = ""
+        self.pseudo = pseudo
+        self.native_sysroot = native_sysroot
+
+    """
+    Update the package manager package database.
+    """
+    @abstractmethod
+    def update(self):
+        pass
+
+    """
+    Install a list of packages. 'pkgs' is a list object. If 'attempt_only' is
+    True, installation failures are ignored.
+    """
+    @abstractmethod
+    def install(self, pkgs, attempt_only=False):
+        pass
+
+    """
+    Remove a list of packages. 'pkgs' is a list object. If 'with_dependencies'
+    is False, the any dependencies are left in place.
+    """
+    @abstractmethod
+    def remove(self, pkgs, with_dependencies=True):
+        pass
+
+    """
+    This function creates the index files
+    """
+    @abstractmethod
+    def write_index(self):
+        pass
+
+    @abstractmethod
+    def remove_packaging_data(self):
+        pass
+
+    @abstractmethod
+    def list_installed(self, format=None):
+        pass
+
+    @abstractmethod
+    def insert_feeds_uris(self):
+        pass
+
+    """
+    Install complementary packages based upon the list of currently installed
+    packages e.g. locales, *-dev, *-dbg, etc. This will only attempt to install
+    these packages, if they don't exist then no error will occur.  Note: every
+    backend needs to call this function explicitly after the normal package
+    installation
+    """
+    def install_complementary(self, globs=None):
+        # we need to write the list of installed packages to a file because the
+        # oe-pkgdata-util reads it from a file
+        if self.d.has_key('WORKDIR'):
+            installed_pkgs_file = os.path.join(self.d['WORKDIR'],
+                                           "installed_pkgs.txt")
+        else:
+            msger.error("No WORKDIR provided!")
+
+        with open(installed_pkgs_file, "w+") as installed_pkgs:
+            installed_pkgs.write(self.list_installed("arch"))
+
+        if globs is None:
+            if self.d.has_key('IMAGE_INSTALL_COMPLEMENTARY'):
+                globs = self.d['IMAGE_INSTALL_COMPLEMENTARY']
+            split_linguas = set()
+
+            if self.d.has_key('IMAGE_LINGUAS'):
+                for translation in self.d['IMAGE_LINGUAS'].split():
+                    split_linguas.add(translation)
+                    split_linguas.add(translation.split('-')[0])
+
+            split_linguas = sorted(split_linguas)
+
+            for lang in split_linguas:
+                globs += " *-locale-%s" % lang
+
+        if globs is None:
+            return
+
+        if not self.d.has_key('PKGDATA_DIR'):
+            msger.error("No PKGDATA_DIR provided!")
+
+        cmd = [which(os.getenv('PATH'), "oe-pkgdata-util"),
+               "glob", self.d['PKGDATA_DIR'], installed_pkgs_file,
+               globs]
+
+        rc, out = exec_native_cmd(self.pseudo + cmd, self.native_sysroot)
+        if rc != 0:
+            msger.error("Could not compute complementary packages list. Command "
+                     "'%s' returned %d" %
+                     (' '.join(cmd), rc))
+
+        self.install(out.split(), attempt_only=True)
+
+
+    def deploy_dir_lock(self):
+        if self.deploy_dir is None:
+            raise RuntimeError("deploy_dir is not set!")
+
+        lock_file_name = os.path.join(self.deploy_dir, "deploy.lock")
+
+        self.deploy_lock = lockfile(lock_file_name)
+
+    def deploy_dir_unlock(self):
+        if self.deploy_lock is None:
+            return
+
+        unlockfile(self.deploy_lock)
+
+        self.deploy_lock = None
+
+
+class WicOpkgPM(WicPackageManager):
+    def __init__(self, d, target_rootfs, config_file, archs, pseudo, native_sysroot, task_name='target'):
+        super(WicOpkgPM, self).__init__(d, pseudo, native_sysroot)
+
+        self.target_rootfs = target_rootfs
+        self.config_file = config_file
+        self.pkg_archs = archs
+        self.task_name = task_name
+
+        if self.d.has_key("DEPLOY_DIR_IPK"):
+            self.deploy_dir = self.d["DEPLOY_DIR_IPK"]
+
+        self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
+        self.opkg_cmd = which(os.getenv('PATH'), "opkg-cl")
+        self.opkg_args = "-f %s -o %s " % (self.config_file, target_rootfs)
+        if self.d.has_key("OPKG_ARGS"):
+            self.opkg_args += self.d["OPKG_ARGS"]
+
+        if self.d.has_key('OPKGLIBDIR'):
+            opkg_lib_dir = self.d['OPKGLIBDIR']
+        else:
+            opkg_lib_dir = ""
+
+        if opkg_lib_dir[0] == "/":
+            opkg_lib_dir = opkg_lib_dir[1:]
+
+        self.opkg_dir = os.path.join(target_rootfs, opkg_lib_dir, "opkg")
+
+        mkdirhier(self.opkg_dir)
+
+        if self.d.has_key("TMPDIR"):
+            tmp_dir = self.d["TMPDIR"]
+        else:
+            tmp_dir = ""
+
+        self.saved_opkg_dir = '%s/saved/%s' % (tmp_dir, self.task_name)
+        if not os.path.exists('%s/saved' % tmp_dir):
+            mkdirhier('%s/saved' % tmp_dir)
+
+        if self.d.has_key('BUILD_IMAGES_FROM_FEEDS') and self.d['BUILD_IMAGES_FROM_FEEDS'] != "1":
+            self._create_config()
+        else:
+            self._create_custom_config()
+
+        self.indexer = WicOpkgIndexer(self.d, self.deploy_dir)
+
+    """
+    This function will change a package's status in /var/lib/opkg/status file.
+    If 'packages' is None then the new_status will be applied to all
+    packages
+    """
+    def mark_packages(self, status_tag, packages=None):
+        status_file = os.path.join(self.opkg_dir, "status")
+
+        with open(status_file, "r") as sf:
+            with open(status_file + ".tmp", "w+") as tmp_sf:
+                if packages is None:
+                    tmp_sf.write(re.sub(r"Package: (.*?)\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)",
+                                        r"Package: \1\n\2Status: \3%s" % status_tag,
+                                        sf.read()))
+                else:
+                    if type(packages).__name__ != "list":
+                        raise TypeError("'packages' should be a list object")
+
+                    status = sf.read()
+                    for pkg in packages:
+                        status = re.sub(r"Package: %s\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)" % pkg,
+                                        r"Package: %s\n\1Status: \2%s" % (pkg, status_tag),
+                                        status)
+
+                    tmp_sf.write(status)
+
+        os.rename(status_file + ".tmp", status_file)
+
+    def _create_custom_config(self):
+        msger.info("Building from feeds activated!")
+
+        with open(self.config_file, "w+") as config_file:
+            priority = 1
+            for arch in self.pkg_archs.split():
+                config_file.write("arch %s %d\n" % (arch, priority))
+                priority += 5
+
+            if self.d.has_key('IPK_FEED_URIS'):
+                ipk_feed_uris = self.d['IPK_FEED_URIS']
+            else:
+                ipk_feed_uris = ""
+
+            for line in ipk_feed_uris.split():
+                feed_match = re.match("^[ \t]*(.*)##([^ \t]*)[ \t]*$", line)
+
+                if feed_match is not None:
+                    feed_name = feed_match.group(1)
+                    feed_uri = feed_match.group(2)
+
+                    msger.info("Add %s feed with URL %s" % (feed_name, feed_uri))
+
+                    config_file.write("src/gz %s %s\n" % (feed_name, feed_uri))
+
+            """
+            Allow to use package deploy directory contents as quick devel-testing
+            feed. This creates individual feed configs for each arch subdir of those
+            specified as compatible for the current machine.
+            NOTE: Development-helper feature, NOT a full-fledged feed.
+            """
+            if self.d.has_key('FEED_DEPLOYDIR_BASE_URI'):
+                feed_deploydir_base_dir = self.d['FEED_DEPLOYDIR_BASE_URI']
+            else:
+                feed_deploydir_base_dir = ""
+
+            if feed_deploydir_base_dir != "":
+                for arch in self.pkg_archs.split():
+                    if self.d.has_key("sysconfdir"):
+                        sysconfdir = self.d["sysconfdir"]
+                    else:
+                        sysconfdir = None
+
+                    cfg_file_name = os.path.join(self.target_rootfs,
+                                                 sysconfdir,
+                                                 "opkg",
+                                                 "local-%s-feed.conf" % arch)
+
+                    with open(cfg_file_name, "w+") as cfg_file:
+                        cfg_file.write("src/gz local-%s %s/%s" %
+                                       arch,
+                                       feed_deploydir_base_dir,
+                                       arch)
+
+    def _create_config(self):
+        with open(self.config_file, "w+") as config_file:
+            priority = 1
+            for arch in self.pkg_archs.split():
+                config_file.write("arch %s %d\n" % (arch, priority))
+                priority += 5
+
+            config_file.write("src oe file:%s\n" % self.deploy_dir)
+
+            for arch in self.pkg_archs.split():
+                pkgs_dir = os.path.join(self.deploy_dir, arch)
+                if os.path.isdir(pkgs_dir):
+                    config_file.write("src oe-%s file:%s\n" %
+                                      (arch, pkgs_dir))
+
+    def insert_feeds_uris(self):
+        if self.feed_uris == "":
+            return
+
+        rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf'
+                                  % self.target_rootfs)
+
+        with open(rootfs_config, "w+") as config_file:
+            uri_iterator = 0
+            for uri in self.feed_uris.split():
+                config_file.write("src/gz url-%d %s/ipk\n" %
+                                  (uri_iterator, uri))
+
+                for arch in self.pkg_archs.split():
+                    if not os.path.exists(os.path.join(self.deploy_dir, arch)):
+                        continue
+                    msger.info('Note: adding opkg channel url-%s-%d (%s)' %
+                        (arch, uri_iterator, uri))
+
+                    config_file.write("src/gz uri-%s-%d %s/ipk/%s\n" %
+                                      (arch, uri_iterator, uri, arch))
+                uri_iterator += 1
+
+    def update(self):
+        self.deploy_dir_lock()
+
+        cmd = "%s %s update" % (self.opkg_cmd, self.opkg_args)
+
+        rc, out = exec_native_cmd(self.pseudo + cmd, self.native_sysroot)
+        if rc != 0:
+            self.deploy_dir_unlock()
+            msger.error("Unable to update the package index files. Command '%s' "
+                     "returned %d" % (cmd, rc))
+
+        self.deploy_dir_unlock()
+
+    def install(self, pkgs, attempt_only=False):
+        if attempt_only and len(pkgs) == 0:
+            return
+
+        cmd = "%s %s install %s" % (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
+
+        os.environ['D'] = self.target_rootfs
+        os.environ['OFFLINE_ROOT'] = self.target_rootfs
+        os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs
+        os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs
+        if self.d.has_key('WORKDIR'):
+            os.environ['INTERCEPT_DIR'] = os.path.join(self.d['WORKDIR'],
+                                                   "intercept_scripts")
+        else:
+            os.environ['INTERCEPT_DIR'] = "."
+            msger.warning("No WORKDIR provided!")
+
+        if self.d.has_key('STAGING_DIR_NATIVE'):
+            os.environ['NATIVE_ROOT'] = self.d['STAGING_DIR_NATIVE']
+        else:
+            msger.error("No STAGING_DIR_NATIVE provided!")
+
+        rc, out = exec_native_cmd(self.pseudo + cmd, self.native_sysroot)
+        if rc != 0:
+            msger.error("Unable to install packages. "
+                        "Command '%s' returned %d" % (cmd, rc))
+
+
+    def remove(self, pkgs, with_dependencies=True):
+        if with_dependencies:
+            cmd = "%s %s --force-depends --force-remove --force-removal-of-dependent-packages remove %s" % \
+                (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
+        else:
+            cmd = "%s %s --force-depends remove %s" % \
+                (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
+
+        rc, out = exec_native_cmd(self.pseudo + cmd, self.native_sysroot)
+        if rc != 0:
+            msger.error("Unable to remove packages. Command '%s' "
+                     "returned %d" % (cmd, rc))
+
+
+    def write_index(self):
+        self.deploy_dir_lock()
+
+        result = self.indexer.write_index()
+
+        self.deploy_dir_unlock()
+
+        if result is not None:
+            msger.error(result)
+
+    def remove_packaging_data(self):
+        remove(self.opkg_dir, True)
+        # create the directory back, it's needed by PM lock
+        mkdirhier(self.opkg_dir)
+
+    def list_installed(self, format=None):
+        return WicOpkgPkgsList(self.d, self.target_rootfs, self.config_file).list(format)
+
+    def handle_bad_recommendations(self):
+        if self.d.has_key("BAD_RECOMMENDATIONS"):
+            bad_recommendations = self.d["BAD_RECOMMENDATIONS"]
+        else:
+            bad_recommendations = ""
+
+        if bad_recommendations.strip() == "":
+            return
+
+        status_file = os.path.join(self.opkg_dir, "status")
+
+        # If status file existed, it means the bad recommendations has already
+        # been handled
+        if os.path.exists(status_file):
+            return
+
+        cmd = "%s %s info " % (self.opkg_cmd, self.opkg_args)
+
+        with open(status_file, "w+") as status:
+            for pkg in bad_recommendations.split():
+                pkg_info = cmd + pkg
+
+                try:
+                    output = subprocess.check_output(pkg_info.split(), stderr=subprocess.STDOUT).strip()
+                except subprocess.CalledProcessError as e:
+                    msger.error("Cannot get package info. Command '%s' "
+                             "returned %d:\n%s" % (pkg_info, e.returncode, e.output))
+
+                if output == "":
+                    msger.info("Ignored bad recommendation: '%s' is "
+                            "not a package" % pkg)
+                    continue
+
+                for line in output.split('\n'):
+                    if line.startswith("Status:"):
+                        status.write("Status: deinstall hold not-installed\n")
+                    else:
+                        status.write(line + "\n")
+
+    '''
+    The following function dummy installs pkgs and returns the log of output.
+    '''
+    def dummy_install(self, pkgs):
+        if len(pkgs) == 0:
+            return
+
+        # Create an temp dir as opkg root for dummy installation
+        if self.d.has_key("TMPDIR"):
+            tmp_dir = self.d["TMPDIR"]
+        else:
+            tmp_dir = "."
+            msger.warning("No TMPDIR provided!")
+
+        temp_rootfs = '%s/opkg' % tmp_dir
+        temp_opkg_dir = os.path.join(temp_rootfs, 'var/lib/opkg')
+        mkdirhier(temp_opkg_dir)
+
+        opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs)
+        if self.d.has_key("OPKG_ARGS"):
+            opkg_args += self.d["OPKG_ARGS"]
+
+        cmd = "%s %s update" % (self.opkg_cmd, opkg_args)
+        try:
+            subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+        except subprocess.CalledProcessError as e:
+            msger.error("Unable to update. Command '%s' "
+                     "returned %d:\n%s" % (cmd, e.returncode, e.output))
+
+        # Dummy installation
+        cmd = "%s %s --noaction install %s " % (self.opkg_cmd,
+                                                opkg_args,
+                                                ' '.join(pkgs))
+        try:
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+        except subprocess.CalledProcessError as e:
+            msger.error("Unable to dummy install packages. Command '%s' "
+                     "returned %d:\n%s" % (cmd, e.returncode, e.output))
+
+        remove(temp_rootfs, True)
+
+        return output
+
+    def backup_packaging_data(self):
+        # Save the opkglib for increment ipk image generation
+        if os.path.exists(self.saved_opkg_dir):
+            remove(self.saved_opkg_dir, True)
+        shutil.copytree(self.opkg_dir,
+                        self.saved_opkg_dir,
+                        symlinks=True)
+
+    def recover_packaging_data(self):
+        # Move the opkglib back
+        if os.path.exists(self.saved_opkg_dir):
+            if os.path.exists(self.opkg_dir):
+                remove(self.opkg_dir, True)
+
+            msger.info('Recover packaging data')
+            shutil.copytree(self.saved_opkg_dir,
+                            self.opkg_dir,
+                            symlinks=True)
+
+
+def wic_generate_index_files(d):
+    if d.has_key('PACKAGE_CLASSES'):
+        classes = d['PACKAGE_CLASSES'].replace("package_", "").split()
+    else:
+        classes = ""
+        msger.warning("No PACKAGE_CLASSES provided!")
+
+    if d.has_key('DEPLOY_DIR_IPK'):
+        deploy_dir_ipk = d['DEPLOY_DIR_IPK']
+    else:
+        deploy_dir_ipk = None
+        msger.warning("No DEPLOY_DIR_IPK provided!")
+
+    indexer_map = {
+        "ipk": (WicOpkgIndexer, deploy_dir_ipk)
+    }
+
+    result = None
+
+    for pkg_class in classes:
+        if not pkg_class in indexer_map:
+            continue
+
+        if os.path.exists(indexer_map[pkg_class][1]):
+            result = indexer_map[pkg_class][0](d, indexer_map[pkg_class][1]).write_index()
+
+            if result is not None:
+                msger.error(result)
diff --git a/scripts/lib/wic/utils/partitionedfs.py b/scripts/lib/wic/utils/partitionedfs.py
new file mode 100644
index 0000000000..fb95cc790e
--- /dev/null
+++ b/scripts/lib/wic/utils/partitionedfs.py
@@ -0,0 +1,360 @@
+#!/usr/bin/env python -tt
+#
+# Copyright (c) 2009, 2010, 2011 Intel, Inc.
+# Copyright (c) 2007, 2008 Red Hat, Inc.
+# Copyright (c) 2008 Daniel P. Berrange
+# Copyright (c) 2008 David P. Huff
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os
+
+from wic import msger
+from wic.utils import runner
+from wic.utils.errors import ImageError
+from wic.utils.fs_related import *
+from wic.utils.oe.misc import *
+
+# Overhead of the MBR partitioning scheme (just one sector)
+MBR_OVERHEAD = 1
+
+# Size of a sector in bytes
+SECTOR_SIZE = 512
+
+class Image:
+    """
+    Generic base object for an image.
+
+    An Image is a container for a set of DiskImages and associated
+    partitions.
+    """
+    def __init__(self):
+        self.disks = {}
+        self.partitions = []
+        self.parted = find_binary_path("parted")
+        # Size of a sector used in calculations
+        self.sector_size = SECTOR_SIZE
+        self._partitions_layed_out = False
+
+    def __add_disk(self, disk_name):
+        """ Add a disk 'disk_name' to the internal list of disks. Note,
+        'disk_name' is the name of the disk in the target system
+        (e.g., sdb). """
+
+        if disk_name in self.disks:
+            # We already have this disk
+            return
+
+        assert not self._partitions_layed_out
+
+        self.disks[disk_name] = \
+                { 'disk': None,     # Disk object
+                  'numpart': 0,     # Number of allocate partitions
+                  'partitions': [], # Indexes to self.partitions
+                  'offset': 0,      # Offset of next partition (in sectors)
+                  # Minimum required disk size to fit all partitions (in bytes)
+                  'min_size': 0,
+                  'ptable_format': "msdos" } # Partition table format
+
+    def add_disk(self, disk_name, disk_obj):
+        """ Add a disk object which have to be partitioned. More than one disk
+        can be added. In case of multiple disks, disk partitions have to be
+        added for each disk separately with 'add_partition()". """
+
+        self.__add_disk(disk_name)
+        self.disks[disk_name]['disk'] = disk_obj
+
+    def __add_partition(self, part):
+        """ This is a helper function for 'add_partition()' which adds a
+        partition to the internal list of partitions. """
+
+        assert not self._partitions_layed_out
+
+        self.partitions.append(part)
+        self.__add_disk(part['disk_name'])
+
+    def add_partition(self, size, disk_name, mountpoint, source_file = None, fstype = None,
+                      label=None, fsopts = None, boot = False, align = None,
+                      part_type = None):
+        """ Add the next partition. Prtitions have to be added in the
+        first-to-last order. """
+
+        ks_pnum = len(self.partitions)
+
+        # Converting MB to sectors for parted
+        size = size * 1024 * 1024 / self.sector_size
+
+        # We still need partition for "/" or non-subvolume
+        if mountpoint == "/" or not fsopts:
+            part = { 'ks_pnum' : ks_pnum, # Partition number in the KS file
+                     'size': size, # In sectors
+                     'mountpoint': mountpoint, # Mount relative to chroot
+                     'source_file': source_file, # partition contents
+                     'fstype': fstype, # Filesystem type
+                     'fsopts': fsopts, # Filesystem mount options
+                     'label': label, # Partition label
+                     'disk_name': disk_name, # physical disk name holding partition
+                     'device': None, # kpartx device node for partition
+                     'num': None, # Partition number
+                     'boot': boot, # Bootable flag
+                     'align': align, # Partition alignment
+                     'part_type' : part_type } # Partition type
+
+            self.__add_partition(part)
+
+    def layout_partitions(self, ptable_format = "msdos"):
+        """ Layout the partitions, meaning calculate the position of every
+        partition on the disk. The 'ptable_format' parameter defines the
+        partition table format and may be "msdos". """
+
+        msger.debug("Assigning %s partitions to disks" % ptable_format)
+
+        if ptable_format not in ('msdos'):
+            raise ImageError("Unknown partition table format '%s', supported " \
+                             "formats are: 'msdos'" % ptable_format)
+
+        if self._partitions_layed_out:
+            return
+
+        self._partitions_layed_out = True
+
+        # Go through partitions in the order they are added in .ks file
+        for n in range(len(self.partitions)):
+            p = self.partitions[n]
+
+            if not self.disks.has_key(p['disk_name']):
+                raise ImageError("No disk %s for partition %s" \
+                                 % (p['disk_name'], p['mountpoint']))
+
+            if p['part_type']:
+                # The --part-type can also be implemented for MBR partitions,
+                # in which case it would map to the 1-byte "partition type"
+                # filed at offset 3 of the partition entry.
+                raise ImageError("setting custom partition type is not " \
+                                 "implemented for msdos partitions")
+
+            # Get the disk where the partition is located
+            d = self.disks[p['disk_name']]
+            d['numpart'] += 1
+            d['ptable_format'] = ptable_format
+
+            if d['numpart'] == 1:
+                if ptable_format == "msdos":
+                    overhead = MBR_OVERHEAD
+
+                # Skip one sector required for the partitioning scheme overhead
+                d['offset'] += overhead
+
+            if p['align']:
+                # If not first partition and we do have alignment set we need
+                # to align the partition.
+                # FIXME: This leaves a empty spaces to the disk. To fill the
+                # gaps we could enlargea the previous partition?
+
+                # Calc how much the alignment is off.
+                align_sectors = d['offset'] % (p['align'] * 1024 / self.sector_size)
+
+                if align_sectors:
+                    # If partition is not aligned as required, we need
+                    # to move forward to the next alignment point
+                    align_sectors = (p['align'] * 1024 / self.sector_size) - align_sectors
+
+                    msger.debug("Realignment for %s%s with %s sectors, original"
+                                " offset %s, target alignment is %sK." %
+                                (p['disk_name'], d['numpart'], align_sectors,
+                                 d['offset'], p['align']))
+
+                    # increase the offset so we actually start the partition on right alignment
+                    d['offset'] += align_sectors
+
+            p['start'] = d['offset']
+            d['offset'] += p['size']
+
+            p['type'] = 'primary'
+            p['num'] = d['numpart']
+
+            if d['ptable_format'] == "msdos":
+                if d['numpart'] > 2:
+                    # Every logical partition requires an additional sector for
+                    # the EBR, so steal the last sector from the end of each
+                    # partition starting from the 3rd one for the EBR. This
+                    # will make sure the logical partitions are aligned
+                    # correctly.
+                    p['size'] -= 1
+
+                if d['numpart'] > 3:
+                    p['type'] = 'logical'
+                    p['num'] = d['numpart'] + 1
+
+            d['partitions'].append(n)
+            msger.debug("Assigned %s to %s%d, sectors range %d-%d size %d "
+                        "sectors (%d bytes)." \
+                            % (p['mountpoint'], p['disk_name'], p['num'],
+                               p['start'], p['start'] + p['size'] - 1,
+                               p['size'], p['size'] * self.sector_size))
+
+        # Once all the partitions have been layed out, we can calculate the
+        # minumim disk sizes.
+        for disk_name, d in self.disks.items():
+            d['min_size'] = d['offset']
+
+            d['min_size'] *= self.sector_size
+
+    def __run_parted(self, args):
+        """ Run parted with arguments specified in the 'args' list. """
+
+        args.insert(0, self.parted)
+        msger.debug(args)
+
+        rc, out = runner.runtool(args, catch = 3)
+        out = out.strip()
+        if out:
+            msger.debug('"parted" output: %s' % out)
+
+        if rc != 0:
+            # We don't throw exception when return code is not 0, because
+            # parted always fails to reload part table with loop devices. This
+            # prevents us from distinguishing real errors based on return
+            # code.
+            msger.error("WARNING: parted returned '%s' instead of 0 (use --debug for details)" % rc)
+
+    def __create_partition(self, device, parttype, fstype, start, size):
+        """ Create a partition on an image described by the 'device' object. """
+
+        # Start is included to the size so we need to substract one from the end.
+        end = start + size - 1
+        msger.debug("Added '%s' partition, sectors %d-%d, size %d sectors" %
+                    (parttype, start, end, size))
+
+        args = ["-s", device, "unit", "s", "mkpart", parttype]
+        if fstype:
+            args.extend([fstype])
+        args.extend(["%d" % start, "%d" % end])
+
+        return self.__run_parted(args)
+
+    def __format_disks(self):
+        self.layout_partitions()
+
+        for dev in self.disks.keys():
+            d = self.disks[dev]
+            msger.debug("Initializing partition table for %s" % \
+                        (d['disk'].device))
+            self.__run_parted(["-s", d['disk'].device, "mklabel",
+                               d['ptable_format']])
+
+        msger.debug("Creating partitions")
+
+        for p in self.partitions:
+            d = self.disks[p['disk_name']]
+            if d['ptable_format'] == "msdos" and p['num'] == 5:
+                # The last sector of the 3rd partition was reserved for the EBR
+                # of the first _logical_ partition. This is why the extended
+                # partition should start one sector before the first logical
+                # partition.
+                self.__create_partition(d['disk'].device, "extended",
+                                        None, p['start'] - 1,
+                                        d['offset'] - p['start'])
+
+            if p['fstype'] == "swap":
+                parted_fs_type = "linux-swap"
+            elif p['fstype'] == "vfat":
+                parted_fs_type = "fat32"
+            elif p['fstype'] == "msdos":
+                parted_fs_type = "fat16"
+            else:
+                # Type for ext2/ext3/ext4/btrfs
+                parted_fs_type = "ext2"
+
+            # Boot ROM of OMAP boards require vfat boot partition to have an
+            # even number of sectors.
+            if p['mountpoint'] == "/boot" and p['fstype'] in ["vfat", "msdos"] \
+               and p['size'] % 2:
+                msger.debug("Substracting one sector from '%s' partition to " \
+                            "get even number of sectors for the partition" % \
+                            p['mountpoint'])
+                p['size'] -= 1
+
+            self.__create_partition(d['disk'].device, p['type'],
+                                    parted_fs_type, p['start'], p['size'])
+
+            if p['boot']:
+                flag_name = "boot"
+                msger.debug("Set '%s' flag for partition '%s' on disk '%s'" % \
+                            (flag_name, p['num'], d['disk'].device))
+                self.__run_parted(["-s", d['disk'].device, "set",
+                                   "%d" % p['num'], flag_name, "on"])
+
+            # Parted defaults to enabling the lba flag for fat16 partitions,
+            # which causes compatibility issues with some firmware (and really
+            # isn't necessary).
+            if parted_fs_type == "fat16":
+                if d['ptable_format'] == 'msdos':
+                    msger.debug("Disable 'lba' flag for partition '%s' on disk '%s'" % \
+                                (p['num'], d['disk'].device))
+                    self.__run_parted(["-s", d['disk'].device, "set",
+                                       "%d" % p['num'], "lba", "off"])
+
+    def cleanup(self):
+        if self.disks:
+            for dev in self.disks.keys():
+                d = self.disks[dev]
+                try:
+                    d['disk'].cleanup()
+                except:
+                    pass
+
+    def __write_partition(self, num, source_file, start, size):
+        """
+        Install source_file contents into a partition.
+        """
+        if not source_file: # nothing to write
+            return
+
+        # Start is included in the size so need to substract one from the end.
+        end = start + size - 1
+        msger.debug("Installed %s in partition %d, sectors %d-%d, size %d sectors" % (source_file, num, start, end, size))
+
+        dd_cmd = "dd if=%s of=%s bs=%d seek=%d count=%d conv=notrunc" % \
+            (source_file, self.image_file, self.sector_size, start, size)
+        exec_cmd(dd_cmd)
+
+
+    def assemble(self, image_file):
+        msger.debug("Installing partitions")
+
+        self.image_file = image_file
+
+        for p in self.partitions:
+            d = self.disks[p['disk_name']]
+            if d['ptable_format'] == "msdos" and p['num'] == 5:
+                # The last sector of the 3rd partition was reserved for the EBR
+                # of the first _logical_ partition. This is why the extended
+                # partition should start one sector before the first logical
+                # partition.
+                self.__write_partition(p['num'], p['source_file'],
+                                       p['start'] - 1,
+                                       d['offset'] - p['start'])
+
+            self.__write_partition(p['num'], p['source_file'],
+                                   p['start'], p['size'])
+
+    def create(self):
+        for dev in self.disks.keys():
+            d = self.disks[dev]
+            d['disk'].create()
+
+        self.__format_disks()
+
+        return
diff --git a/scripts/lib/wic/utils/runner.py b/scripts/lib/wic/utils/runner.py
new file mode 100644
index 0000000000..2ae9f417c5
--- /dev/null
+++ b/scripts/lib/wic/utils/runner.py
@@ -0,0 +1,109 @@
+#!/usr/bin/env python -tt
+#
+# Copyright (c) 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os
+import subprocess
+
+from wic import msger
+
+def runtool(cmdln_or_args, catch=1):
+    """ wrapper for most of the subprocess calls
+    input:
+        cmdln_or_args: can be both args and cmdln str (shell=True)
+        catch: 0, quitely run
+               1, only STDOUT
+               2, only STDERR
+               3, both STDOUT and STDERR
+    return:
+        (rc, output)
+        if catch==0: the output will always None
+    """
+
+    if catch not in (0, 1, 2, 3):
+        # invalid catch selection, will cause exception, that's good
+        return None
+
+    if isinstance(cmdln_or_args, list):
+        cmd = cmdln_or_args[0]
+        shell = False
+    else:
+        import shlex
+        cmd = shlex.split(cmdln_or_args)[0]
+        shell = True
+
+    if catch != 3:
+        dev_null = os.open("/dev/null", os.O_WRONLY)
+
+    if catch == 0:
+        sout = dev_null
+        serr = dev_null
+    elif catch == 1:
+        sout = subprocess.PIPE
+        serr = dev_null
+    elif catch == 2:
+        sout = dev_null
+        serr = subprocess.PIPE
+    elif catch == 3:
+        sout = subprocess.PIPE
+        serr = subprocess.STDOUT
+
+    try:
+        p = subprocess.Popen(cmdln_or_args, stdout=sout,
+                             stderr=serr, shell=shell)
+        (sout, serr) = p.communicate()
+        # combine stdout and stderr, filter None out
+        out = ''.join(filter(None, [sout, serr]))
+    except OSError, e:
+        if e.errno == 2:
+            # [Errno 2] No such file or directory
+            msger.error('Cannot run command: %s, lost dependency?' % cmd)
+        else:
+            raise # relay
+    finally:
+        if catch != 3:
+            os.close(dev_null)
+
+    return (p.returncode, out)
+
+def show(cmdln_or_args):
+    # show all the message using msger.verbose
+
+    rc, out = runtool(cmdln_or_args, catch=3)
+
+    if isinstance(cmdln_or_args, list):
+        cmd = ' '.join(cmdln_or_args)
+    else:
+        cmd = cmdln_or_args
+
+    msg =  'running command: "%s"' % cmd
+    if out: out = out.strip()
+    if out:
+        msg += ', with output::'
+        msg += '\n  +----------------'
+        for line in out.splitlines():
+            msg += '\n  | %s' % line
+        msg += '\n  +----------------'
+
+    msger.verbose(msg)
+    return rc
+
+def outs(cmdln_or_args, catch=1):
+    # get the outputs of tools
+    return runtool(cmdln_or_args, catch)[1].strip()
+
+def quiet(cmdln_or_args):
+    return runtool(cmdln_or_args, catch=0)[0]