wic: Add mic w/pykickstart

This is the starting point for the implemention described in [YOCTO
3847] which came to the conclusion that it would make sense to use
kickstart syntax to implement image creation in OpenEmbedded.  I
subsequently realized that there was an existing tool that already
implemented image creation using kickstart syntax, the Tizen/Meego mic
tool.  As such, it made sense to use that as a starting point - this
commit essentially just copies the relevant Python code from the MIC
tool to the scripts/lib dir, where it can be accessed by the
previously created wic tool.

Most of this will be removed or renamed by later commits, since we're
initially focusing on partitioning only.  Care should be taken so that
we can easily add back any additional functionality should we decide
later to expand the tool, though (we may also want to contribute our
local changes to the mic tool to the Tizen project if it makes sense,
and therefore should avoid gratuitous changes to the original code if
possible).

Added the /mic subdir from Tizen mic repo as a starting point:

 git clone git://review.tizen.org/tools/mic.git

 For reference, the top commit:

 commit 20164175ddc234a17b8a12c33d04b012347b1530
 Author: Gui Chen <gui.chen@intel.com>
 Date:   Sun Jun 30 22:32:16 2013 -0400

    bump up to 0.19.2

Also added the /plugins subdir, moved to under the /mic subdir (to
match the default plugin_dir location in mic.conf.in, which was
renamed to yocto-image.conf (moved and renamed by later patches) and
put into /scripts.

(From OE-Core rev: 31f0360f1fd4ebc9dfcaed42d1c50d2448b4632e)

Signed-off-by: Tom Zanussi <tom.zanussi@linux.intel.com>
Signed-off-by: Saul Wold <sgw@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 1067 insertions, 0 deletions

diff --git a/scripts/lib/mic/utils/misc.py b/scripts/lib/mic/utils/misc.py
new file mode 100644
index 0000000000..63024346a9
--- /dev/null
+++ b/scripts/lib/mic/utils/misc.py
@@ -0,0 +1,1067 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2010, 2011 Intel Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os
+import sys
+import time
+import tempfile
+import re
+import shutil
+import glob
+import hashlib
+import subprocess
+import platform
+import traceback
+
+
+try:
+    import sqlite3 as sqlite
+except ImportError:
+    import sqlite
+
+try:
+    from xml.etree import cElementTree
+except ImportError:
+    import cElementTree
+xmlparse = cElementTree.parse
+
+from mic import msger
+from mic.utils.errors import CreatorError, SquashfsError
+from mic.utils.fs_related import find_binary_path, makedirs
+from mic.utils.grabber import myurlgrab
+from mic.utils.proxy import get_proxy_for
+from mic.utils import runner
+from mic.utils import rpmmisc
+
+
+RPM_RE  = re.compile("(.*)\.(.*) (.*)-(.*)")
+RPM_FMT = "%(name)s.%(arch)s %(version)s-%(release)s"
+SRPM_RE = re.compile("(.*)-(\d+.*)-(\d+\.\d+).src.rpm")
+
+
+def build_name(kscfg, release=None, prefix = None, suffix = None):
+    """Construct and return an image name string.
+
+    This is a utility function to help create sensible name and fslabel
+    strings. The name is constructed using the sans-prefix-and-extension
+    kickstart filename and the supplied prefix and suffix.
+
+    kscfg -- a path to a kickstart file
+    release --  a replacement to suffix for image release
+    prefix -- a prefix to prepend to the name; defaults to None, which causes
+              no prefix to be used
+    suffix -- a suffix to append to the name; defaults to None, which causes
+              a YYYYMMDDHHMM suffix to be used
+
+    Note, if maxlen is less then the len(suffix), you get to keep both pieces.
+
+    """
+    name = os.path.basename(kscfg)
+    idx = name.rfind('.')
+    if idx >= 0:
+        name = name[:idx]
+
+    if release is not None:
+        suffix = ""
+    if prefix is None:
+        prefix = ""
+    if suffix is None:
+        suffix = time.strftime("%Y%m%d%H%M")
+
+    if name.startswith(prefix):
+        name = name[len(prefix):]
+
+    prefix = "%s-" % prefix if prefix else ""
+    suffix = "-%s" % suffix if suffix else ""
+
+    ret = prefix + name + suffix
+    return ret
+
+def get_distro():
+    """Detect linux distribution, support "meego"
+    """
+
+    support_dists = ('SuSE',
+                     'debian',
+                     'fedora',
+                     'redhat',
+                     'centos',
+                     'meego',
+                     'moblin',
+                     'tizen')
+    try:
+        (dist, ver, id) = platform.linux_distribution( \
+                              supported_dists = support_dists)
+    except:
+        (dist, ver, id) = platform.dist( \
+                              supported_dists = support_dists)
+
+    return (dist, ver, id)
+
+def get_distro_str():
+    """Get composited string for current linux distribution
+    """
+    (dist, ver, id) = get_distro()
+
+    if not dist:
+        return 'Unknown Linux Distro'
+    else:
+        distro_str = ' '.join(map(str.strip, (dist, ver, id)))
+        return distro_str.strip()
+
+_LOOP_RULE_PTH = None
+
+def hide_loopdev_presentation():
+    udev_rules = "80-prevent-loop-present.rules"
+    udev_rules_dir = [
+                       '/usr/lib/udev/rules.d/',
+                       '/lib/udev/rules.d/',
+                       '/etc/udev/rules.d/'
+                     ]
+
+    global _LOOP_RULE_PTH
+
+    for rdir in udev_rules_dir:
+        if os.path.exists(rdir):
+            _LOOP_RULE_PTH = os.path.join(rdir, udev_rules)
+
+    if not _LOOP_RULE_PTH:
+        return
+
+    try:
+        with open(_LOOP_RULE_PTH, 'w') as wf:
+            wf.write('KERNEL=="loop*", ENV{UDISKS_PRESENTATION_HIDE}="1"')
+
+        runner.quiet('udevadm trigger')
+    except:
+        pass
+
+def unhide_loopdev_presentation():
+    global _LOOP_RULE_PTH
+
+    if not _LOOP_RULE_PTH:
+        return
+
+    try:
+        os.unlink(_LOOP_RULE_PTH)
+        runner.quiet('udevadm trigger')
+    except:
+        pass
+
+def extract_rpm(rpmfile, targetdir):
+    rpm2cpio = find_binary_path("rpm2cpio")
+    cpio = find_binary_path("cpio")
+
+    olddir = os.getcwd()
+    os.chdir(targetdir)
+
+    msger.verbose("Extract rpm file with cpio: %s" % rpmfile)
+    p1 = subprocess.Popen([rpm2cpio, rpmfile], stdout=subprocess.PIPE)
+    p2 = subprocess.Popen([cpio, "-idv"], stdin=p1.stdout,
+                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    (sout, serr) = p2.communicate()
+    msger.verbose(sout or serr)
+
+    os.chdir(olddir)
+
+def compressing(fpath, method):
+    comp_map = {
+        "gz": "gzip",
+        "bz2": "bzip2"
+    }
+    if method not in comp_map:
+        raise CreatorError("Unsupport compress format: %s, valid values: %s"
+                           % (method, ','.join(comp_map.keys())))
+    cmd = find_binary_path(comp_map[method])
+    rc = runner.show([cmd, "-f", fpath])
+    if rc:
+        raise CreatorError("Failed to %s file: %s" % (comp_map[method], fpath))
+
+def taring(dstfile, target):
+    import tarfile
+    basen, ext = os.path.splitext(dstfile)
+    comp = {".tar": None,
+            ".gz": "gz", # for .tar.gz
+            ".bz2": "bz2", # for .tar.bz2
+            ".tgz": "gz",
+            ".tbz": "bz2"}[ext]
+
+    # specify tarball file path
+    if not comp:
+        tarpath = dstfile
+    elif basen.endswith(".tar"):
+        tarpath = basen
+    else:
+        tarpath = basen + ".tar"
+    wf = tarfile.open(tarpath, 'w')
+
+    if os.path.isdir(target):
+        for item in os.listdir(target):
+            wf.add(os.path.join(target, item), item)
+    else:
+        wf.add(target, os.path.basename(target))
+    wf.close()
+
+    if comp:
+        compressing(tarpath, comp)
+        # when dstfile ext is ".tgz" and ".tbz", should rename
+        if not basen.endswith(".tar"):
+            shutil.move("%s.%s" % (tarpath, comp), dstfile)
+
+def ziping(dstfile, target):
+    import zipfile
+    wf = zipfile.ZipFile(dstfile, 'w', compression=zipfile.ZIP_DEFLATED)
+    if os.path.isdir(target):
+        for item in os.listdir(target):
+            fpath = os.path.join(target, item)
+            if not os.path.isfile(fpath):
+                continue
+            wf.write(fpath, item, zipfile.ZIP_DEFLATED)
+    else:
+        wf.write(target, os.path.basename(target), zipfile.ZIP_DEFLATED)
+    wf.close()
+
+pack_formats = {
+    ".tar": taring,
+    ".tar.gz": taring,
+    ".tar.bz2": taring,
+    ".tgz": taring,
+    ".tbz": taring,
+    ".zip": ziping,
+}
+
+def packing(dstfile, target):
+    (base, ext) = os.path.splitext(dstfile)
+    if ext in (".gz", ".bz2") and base.endswith(".tar"):
+        ext = ".tar" + ext
+    if ext not in pack_formats:
+        raise CreatorError("Unsupport pack format: %s, valid values: %s"
+                           % (ext, ','.join(pack_formats.keys())))
+    func = pack_formats[ext]
+    # func should be callable
+    func(dstfile, target)
+
+def human_size(size):
+    """Return human readable string for Bytes size
+    """
+
+    if size <= 0:
+        return "0M"
+    import math
+    measure = ['B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']
+    expo = int(math.log(size, 1024))
+    mant = float(size/math.pow(1024, expo))
+    return "{0:.1f}{1:s}".format(mant, measure[expo])
+
+def get_block_size(file_obj):
+    """ Returns block size for file object 'file_obj'. Errors are indicated by
+    the 'IOError' exception. """
+
+    from fcntl import ioctl
+    import struct
+
+    # Get the block size of the host file-system for the image file by calling
+    # the FIGETBSZ ioctl (number 2).
+    binary_data = ioctl(file_obj, 2, struct.pack('I', 0))
+    return struct.unpack('I', binary_data)[0]
+
+def check_space_pre_cp(src, dst):
+    """Check whether disk space is enough before 'cp' like
+    operations, else exception will be raised.
+    """
+
+    srcsize  = get_file_size(src) * 1024 * 1024
+    freesize = get_filesystem_avail(dst)
+    if srcsize > freesize:
+        raise CreatorError("space on %s(%s) is not enough for about %s files"
+                           % (dst, human_size(freesize), human_size(srcsize)))
+
+def calc_hashes(file_path, hash_names, start = 0, end = None):
+    """ Calculate hashes for a file. The 'file_path' argument is the file
+    to calculate hash functions for, 'start' and 'end' are the starting and
+    ending file offset to calculate the has functions for. The 'hash_names'
+    argument is a list of hash names to calculate. Returns the the list
+    of calculated hash values in the hexadecimal form in the same order
+    as 'hash_names'.
+    """
+    if end == None:
+        end = os.path.getsize(file_path)
+
+    chunk_size = 65536
+    to_read = end - start
+    read = 0
+
+    hashes = []
+    for hash_name in hash_names:
+        hashes.append(hashlib.new(hash_name))
+
+    with open(file_path, "rb") as f:
+        f.seek(start)
+
+        while read < to_read:
+            if read + chunk_size > to_read:
+                chunk_size = to_read - read
+            chunk = f.read(chunk_size)
+            for hash_obj in hashes:
+                hash_obj.update(chunk)
+            read += chunk_size
+
+    result = []
+    for hash_obj in hashes:
+        result.append(hash_obj.hexdigest())
+
+    return result
+
+def get_md5sum(fpath):
+    return calc_hashes(fpath, ('md5', ))[0]
+
+
+def normalize_ksfile(ksconf, release, arch):
+    '''
+    Return the name of a normalized ks file in which macro variables
+    @BUILD_ID@ and @ARCH@ are replace with real values.
+
+    The original ks file is returned if no special macro is used, otherwise
+    a temp file is created and returned, which will be deleted when program
+    exits normally.
+    '''
+
+    if not release:
+        release = "latest"
+    if not arch or re.match(r'i.86', arch):
+        arch = "ia32"
+
+    with open(ksconf) as f:
+        ksc = f.read()
+
+    if "@ARCH@" not in ksc and "@BUILD_ID@" not in ksc:
+        return ksconf
+
+    msger.info("Substitute macro variable @BUILD_ID@/@ARCH@ in ks: %s" % ksconf)
+    ksc = ksc.replace("@ARCH@", arch)
+    ksc = ksc.replace("@BUILD_ID@", release)
+
+    fd, ksconf = tempfile.mkstemp(prefix=os.path.basename(ksconf))
+    os.write(fd, ksc)
+    os.close(fd)
+
+    msger.debug('normalized ks file:%s' % ksconf)
+
+    def remove_temp_ks():
+        try:
+            os.unlink(ksconf)
+        except OSError, err:
+            msger.warning('Failed to remove temp ks file:%s:%s' % (ksconf, err))
+
+    import atexit
+    atexit.register(remove_temp_ks)
+
+    return ksconf
+
+
+def _check_mic_chroot(rootdir):
+    def _path(path):
+        return rootdir.rstrip('/') + path
+
+    release_files = map(_path, [ "/etc/moblin-release",
+                                 "/etc/meego-release",
+                                 "/etc/tizen-release"])
+
+    if not any(map(os.path.exists, release_files)):
+        msger.warning("Dir %s is not a MeeGo/Tizen chroot env" % rootdir)
+
+    if not glob.glob(rootdir + "/boot/vmlinuz-*"):
+        msger.warning("Failed to find kernel module under %s" % rootdir)
+
+    return
+
+def selinux_check(arch, fstypes):
+    try:
+        getenforce = find_binary_path('getenforce')
+    except CreatorError:
+        return
+
+    selinux_status = runner.outs([getenforce])
+    if arch and arch.startswith("arm") and selinux_status == "Enforcing":
+        raise CreatorError("Can't create arm image if selinux is enabled, "
+                           "please run 'setenforce 0' to disable selinux")
+
+    use_btrfs = filter(lambda typ: typ == 'btrfs', fstypes)
+    if use_btrfs and selinux_status == "Enforcing":
+        raise CreatorError("Can't create btrfs image if selinux is enabled,"
+                           " please run 'setenforce 0' to disable selinux")
+
+def get_image_type(path):
+    def _get_extension_name(path):
+        match = re.search("(?<=\.)\w+$", path)
+        if match:
+            return match.group(0)
+        else:
+            return None
+
+    if os.path.isdir(path):
+        _check_mic_chroot(path)
+        return "fs"
+
+    maptab = {
+              "tar": "loop",
+              "raw":"raw",
+              "vmdk":"vmdk",
+              "vdi":"vdi",
+              "iso":"livecd",
+              "usbimg":"liveusb",
+             }
+
+    extension = _get_extension_name(path)
+    if extension in maptab:
+        return maptab[extension]
+
+    fd = open(path, "rb")
+    file_header = fd.read(1024)
+    fd.close()
+    vdi_flag = "<<< Sun VirtualBox Disk Image >>>"
+    if file_header[0:len(vdi_flag)] == vdi_flag:
+        return maptab["vdi"]
+
+    output = runner.outs(['file', path])
+    isoptn = re.compile(r".*ISO 9660 CD-ROM filesystem.*(bootable).*")
+    usbimgptn = re.compile(r".*x86 boot sector.*active.*")
+    rawptn = re.compile(r".*x86 boot sector.*")
+    vmdkptn = re.compile(r".*VMware. disk image.*")
+    ext3fsimgptn = re.compile(r".*Linux.*ext3 filesystem data.*")
+    ext4fsimgptn = re.compile(r".*Linux.*ext4 filesystem data.*")
+    btrfsimgptn = re.compile(r".*BTRFS.*")
+    if isoptn.match(output):
+        return maptab["iso"]
+    elif usbimgptn.match(output):
+        return maptab["usbimg"]
+    elif rawptn.match(output):
+        return maptab["raw"]
+    elif vmdkptn.match(output):
+        return maptab["vmdk"]
+    elif ext3fsimgptn.match(output):
+        return "ext3fsimg"
+    elif ext4fsimgptn.match(output):
+        return "ext4fsimg"
+    elif btrfsimgptn.match(output):
+        return "btrfsimg"
+    else:
+        raise CreatorError("Cannot detect the type of image: %s" % path)
+
+
+def get_file_size(filename):
+    """ Return size in MB unit """
+    cmd = ['du', "-s", "-b", "-B", "1M", filename]
+    rc, duOutput  = runner.runtool(cmd)
+    if rc != 0:
+        raise CreatorError("Failed to run: %s" % ' '.join(cmd))
+    size1 = int(duOutput.split()[0])
+
+    cmd = ['du', "-s", "-B", "1M", filename]
+    rc, duOutput = runner.runtool(cmd)
+    if rc != 0:
+        raise CreatorError("Failed to run: %s" % ' '.join(cmd))
+
+    size2 = int(duOutput.split()[0])
+    return max(size1, size2)
+
+
+def get_filesystem_avail(fs):
+    vfstat = os.statvfs(fs)
+    return vfstat.f_bavail * vfstat.f_bsize
+
+def convert_image(srcimg, srcfmt, dstimg, dstfmt):
+    #convert disk format
+    if dstfmt != "raw":
+        raise CreatorError("Invalid destination image format: %s" % dstfmt)
+    msger.debug("converting %s image to %s" % (srcimg, dstimg))
+    if srcfmt == "vmdk":
+        path = find_binary_path("qemu-img")
+        argv = [path, "convert", "-f", "vmdk", srcimg, "-O", dstfmt,  dstimg]
+    elif srcfmt == "vdi":
+        path = find_binary_path("VBoxManage")
+        argv = [path, "internalcommands", "converttoraw", srcimg, dstimg]
+    else:
+        raise CreatorError("Invalid soure image format: %s" % srcfmt)
+
+    rc = runner.show(argv)
+    if rc == 0:
+        msger.debug("convert successful")
+    if rc != 0:
+        raise CreatorError("Unable to convert disk to %s" % dstfmt)
+
+def uncompress_squashfs(squashfsimg, outdir):
+    """Uncompress file system from squshfs image"""
+    unsquashfs = find_binary_path("unsquashfs")
+    args = [ unsquashfs, "-d", outdir, squashfsimg ]
+    rc = runner.show(args)
+    if (rc != 0):
+        raise SquashfsError("Failed to uncompress %s." % squashfsimg)
+
+def mkdtemp(dir = "/var/tmp", prefix = "mic-tmp-"):
+    """ FIXME: use the dir in mic.conf instead """
+
+    makedirs(dir)
+    return tempfile.mkdtemp(dir = dir, prefix = prefix)
+
+def get_repostrs_from_ks(ks):
+    def _get_temp_reponame(baseurl):
+        md5obj = hashlib.md5(baseurl)
+        tmpreponame = "%s" % md5obj.hexdigest()
+        return tmpreponame
+
+    kickstart_repos = []
+
+    for repodata in ks.handler.repo.repoList:
+        repo = {}
+        for attr in ('name',
+                     'baseurl',
+                     'mirrorlist',
+                     'includepkgs', # val is list
+                     'excludepkgs', # val is list
+                     'cost',    # int
+                     'priority',# int
+                     'save',
+                     'proxy',
+                     'proxyuser',
+                     'proxypasswd',
+                     'proxypasswd',
+                     'debuginfo',
+                     'source',
+                     'gpgkey',
+                     'ssl_verify'):
+            if hasattr(repodata, attr) and getattr(repodata, attr):
+                repo[attr] = getattr(repodata, attr)
+
+        if 'name' not in repo:
+            repo['name'] = _get_temp_reponame(repodata.baseurl)
+
+        kickstart_repos.append(repo)
+
+    return kickstart_repos
+
+def _get_uncompressed_data_from_url(url, filename, proxies):
+    filename = myurlgrab(url, filename, proxies)
+    suffix = None
+    if filename.endswith(".gz"):
+        suffix = ".gz"
+        runner.quiet(['gunzip', "-f", filename])
+    elif filename.endswith(".bz2"):
+        suffix = ".bz2"
+        runner.quiet(['bunzip2', "-f", filename])
+    if suffix:
+        filename = filename.replace(suffix, "")
+    return filename
+
+def _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, filename,
+                            sumtype=None, checksum=None):
+    url = os.path.join(baseurl, filename)
+    filename_tmp = str("%s/%s/%s" % (cachedir, reponame, os.path.basename(filename)))
+    if os.path.splitext(filename_tmp)[1] in (".gz", ".bz2"):
+        filename = os.path.splitext(filename_tmp)[0]
+    else:
+        filename = filename_tmp
+    if sumtype and checksum and os.path.exists(filename):
+        try:
+            sumcmd = find_binary_path("%ssum" % sumtype)
+        except:
+            file_checksum = None
+        else:
+            file_checksum = runner.outs([sumcmd, filename]).split()[0]
+
+        if file_checksum and file_checksum == checksum:
+            return filename
+
+    return _get_uncompressed_data_from_url(url,filename_tmp,proxies)
+
+def get_metadata_from_repos(repos, cachedir):
+    my_repo_metadata = []
+    for repo in repos:
+        reponame = repo['name']
+        baseurl  = repo['baseurl']
+
+
+        if 'proxy' in repo:
+            proxy = repo['proxy']
+        else:
+            proxy = get_proxy_for(baseurl)
+
+        proxies = None
+        if proxy:
+            proxies = {str(baseurl.split(":")[0]):str(proxy)}
+
+        makedirs(os.path.join(cachedir, reponame))
+        url = os.path.join(baseurl, "repodata/repomd.xml")
+        filename = os.path.join(cachedir, reponame, 'repomd.xml')
+        repomd = myurlgrab(url, filename, proxies)
+        try:
+            root = xmlparse(repomd)
+        except SyntaxError:
+            raise CreatorError("repomd.xml syntax error.")
+
+        ns = root.getroot().tag
+        ns = ns[0:ns.rindex("}")+1]
+
+        filepaths = {}
+        checksums = {}
+        sumtypes = {}
+
+        for elm in root.getiterator("%sdata" % ns):
+            if elm.attrib["type"] == "patterns":
+                filepaths['patterns'] = elm.find("%slocation" % ns).attrib['href']
+                checksums['patterns'] = elm.find("%sopen-checksum" % ns).text
+                sumtypes['patterns'] = elm.find("%sopen-checksum" % ns).attrib['type']
+                break
+
+        for elm in root.getiterator("%sdata" % ns):
+            if elm.attrib["type"] in ("group_gz", "group"):
+                filepaths['comps'] = elm.find("%slocation" % ns).attrib['href']
+                checksums['comps'] = elm.find("%sopen-checksum" % ns).text
+                sumtypes['comps'] = elm.find("%sopen-checksum" % ns).attrib['type']
+                break
+
+        primary_type = None
+        for elm in root.getiterator("%sdata" % ns):
+            if elm.attrib["type"] in ("primary_db", "primary"):
+                primary_type = elm.attrib["type"]
+                filepaths['primary'] = elm.find("%slocation" % ns).attrib['href']
+                checksums['primary'] = elm.find("%sopen-checksum" % ns).text
+                sumtypes['primary'] = elm.find("%sopen-checksum" % ns).attrib['type']
+                break
+
+        if not primary_type:
+            continue
+
+        for item in ("primary", "patterns", "comps"):
+            if item not in filepaths:
+                filepaths[item] = None
+                continue
+            if not filepaths[item]:
+                continue
+            filepaths[item] = _get_metadata_from_repo(baseurl,
+                                                      proxies,
+                                                      cachedir,
+                                                      reponame,
+                                                      filepaths[item],
+                                                      sumtypes[item],
+                                                      checksums[item])
+
+        """ Get repo key """
+        try:
+            repokey = _get_metadata_from_repo(baseurl,
+                                              proxies,
+                                              cachedir,
+                                              reponame,
+                                              "repodata/repomd.xml.key")
+        except CreatorError:
+            repokey = None
+            msger.debug("\ncan't get %s/%s" % (baseurl, "repodata/repomd.xml.key"))
+
+        my_repo_metadata.append({"name":reponame,
+                                 "baseurl":baseurl,
+                                 "repomd":repomd,
+                                 "primary":filepaths['primary'],
+                                 "cachedir":cachedir,
+                                 "proxies":proxies,
+                                 "patterns":filepaths['patterns'],
+                                 "comps":filepaths['comps'],
+                                 "repokey":repokey})
+
+    return my_repo_metadata
+
+def get_rpmver_in_repo(repometadata):
+    for repo in repometadata:
+        if repo["primary"].endswith(".xml"):
+            root = xmlparse(repo["primary"])
+            ns = root.getroot().tag
+            ns = ns[0:ns.rindex("}")+1]
+
+            versionlist = []
+            for elm in root.getiterator("%spackage" % ns):
+                if elm.find("%sname" % ns).text == 'rpm':
+                    for node in elm.getchildren():
+                        if node.tag == "%sversion" % ns:
+                            versionlist.append(node.attrib['ver'])
+
+            if versionlist:
+                return reversed(
+                         sorted(
+                           versionlist,
+                           key = lambda ver: map(int, ver.split('.')))).next()
+
+        elif repo["primary"].endswith(".sqlite"):
+            con = sqlite.connect(repo["primary"])
+            for row in con.execute("select version from packages where "
+                                   "name=\"rpm\" ORDER by version DESC"):
+                con.close()
+                return row[0]
+
+    return None
+
+def get_arch(repometadata):
+    archlist = []
+    for repo in repometadata:
+        if repo["primary"].endswith(".xml"):
+            root = xmlparse(repo["primary"])
+            ns = root.getroot().tag
+            ns = ns[0:ns.rindex("}")+1]
+            for elm in root.getiterator("%spackage" % ns):
+                if elm.find("%sarch" % ns).text not in ("noarch", "src"):
+                    arch = elm.find("%sarch" % ns).text
+                    if arch not in archlist:
+                        archlist.append(arch)
+        elif repo["primary"].endswith(".sqlite"):
+            con = sqlite.connect(repo["primary"])
+            for row in con.execute("select arch from packages where arch not in (\"src\", \"noarch\")"):
+                if row[0] not in archlist:
+                    archlist.append(row[0])
+
+            con.close()
+
+    uniq_arch = []
+    for i in range(len(archlist)):
+        if archlist[i] not in rpmmisc.archPolicies.keys():
+            continue
+        need_append = True
+        j = 0
+        while j < len(uniq_arch):
+            if archlist[i] in rpmmisc.archPolicies[uniq_arch[j]].split(':'):
+                need_append = False
+                break
+            if uniq_arch[j] in rpmmisc.archPolicies[archlist[i]].split(':'):
+                if need_append:
+                    uniq_arch[j] = archlist[i]
+                    need_append = False
+                else:
+                    uniq_arch.remove(uniq_arch[j])
+                    continue
+            j += 1
+        if need_append:
+             uniq_arch.append(archlist[i])
+
+    return uniq_arch, archlist
+
+def get_package(pkg, repometadata, arch = None):
+    ver = ""
+    target_repo = None
+    if not arch:
+        arches = []
+    elif arch not in rpmmisc.archPolicies:
+        arches = [arch]
+    else:
+        arches = rpmmisc.archPolicies[arch].split(':')
+        arches.append('noarch')
+
+    for repo in repometadata:
+        if repo["primary"].endswith(".xml"):
+            root = xmlparse(repo["primary"])
+            ns = root.getroot().tag
+            ns = ns[0:ns.rindex("}")+1]
+            for elm in root.getiterator("%spackage" % ns):
+                if elm.find("%sname" % ns).text == pkg:
+                    if elm.find("%sarch" % ns).text in arches:
+                        version = elm.find("%sversion" % ns)
+                        tmpver = "%s-%s" % (version.attrib['ver'], version.attrib['rel'])
+                        if tmpver > ver:
+                            ver = tmpver
+                            location = elm.find("%slocation" % ns)
+                            pkgpath = "%s" % location.attrib['href']
+                            target_repo = repo
+                        break
+        if repo["primary"].endswith(".sqlite"):
+            con = sqlite.connect(repo["primary"])
+            if arch:
+                sql = 'select version, release, location_href from packages ' \
+                      'where name = "%s" and arch IN ("%s")' % \
+                      (pkg, '","'.join(arches))
+                for row in con.execute(sql):
+                    tmpver = "%s-%s" % (row[0], row[1])
+                    if tmpver > ver:
+                        ver = tmpver
+                        pkgpath = "%s" % row[2]
+                        target_repo = repo
+                    break
+            else:
+                sql = 'select version, release, location_href from packages ' \
+                      'where name = "%s"' % pkg
+                for row in con.execute(sql):
+                    tmpver = "%s-%s" % (row[0], row[1])
+                    if tmpver > ver:
+                        ver = tmpver
+                        pkgpath = "%s" % row[2]
+                        target_repo = repo
+                    break
+            con.close()
+    if target_repo:
+        makedirs("%s/packages/%s" % (target_repo["cachedir"], target_repo["name"]))
+        url = os.path.join(target_repo["baseurl"], pkgpath)
+        filename = str("%s/packages/%s/%s" % (target_repo["cachedir"], target_repo["name"], os.path.basename(pkgpath)))
+        if os.path.exists(filename):
+            ret = rpmmisc.checkRpmIntegrity('rpm', filename)
+            if ret == 0:
+                return filename
+
+            msger.warning("package %s is damaged: %s" %
+                          (os.path.basename(filename), filename))
+            os.unlink(filename)
+
+        pkg = myurlgrab(str(url), filename, target_repo["proxies"])
+        return pkg
+    else:
+        return None
+
+def get_source_name(pkg, repometadata):
+
+    def get_bin_name(pkg):
+        m = RPM_RE.match(pkg)
+        if m:
+            return m.group(1)
+        return None
+
+    def get_src_name(srpm):
+        m = SRPM_RE.match(srpm)
+        if m:
+            return m.group(1)
+        return None
+
+    ver = ""
+    target_repo = None
+
+    pkg_name = get_bin_name(pkg)
+    if not pkg_name:
+        return None
+
+    for repo in repometadata:
+        if repo["primary"].endswith(".xml"):
+            root = xmlparse(repo["primary"])
+            ns = root.getroot().tag
+            ns = ns[0:ns.rindex("}")+1]
+            for elm in root.getiterator("%spackage" % ns):
+                if elm.find("%sname" % ns).text == pkg_name:
+                    if elm.find("%sarch" % ns).text != "src":
+                        version = elm.find("%sversion" % ns)
+                        tmpver = "%s-%s" % (version.attrib['ver'], version.attrib['rel'])
+                        if tmpver > ver:
+                            ver = tmpver
+                            fmt = elm.find("%sformat" % ns)
+                            if fmt:
+                                fns = fmt.getchildren()[0].tag
+                                fns = fns[0:fns.rindex("}")+1]
+                                pkgpath = fmt.find("%ssourcerpm" % fns).text
+                                target_repo = repo
+                        break
+
+        if repo["primary"].endswith(".sqlite"):
+            con = sqlite.connect(repo["primary"])
+            for row in con.execute("select version, release, rpm_sourcerpm from packages where name = \"%s\" and arch != \"src\"" % pkg_name):
+                tmpver = "%s-%s" % (row[0], row[1])
+                if tmpver > ver:
+                    pkgpath = "%s" % row[2]
+                    target_repo = repo
+                break
+            con.close()
+    if target_repo:
+        return get_src_name(pkgpath)
+    else:
+        return None
+
+def get_pkglist_in_patterns(group, patterns):
+    found = False
+    pkglist = []
+    try:
+        root = xmlparse(patterns)
+    except SyntaxError:
+        raise SyntaxError("%s syntax error." % patterns)
+
+    for elm in list(root.getroot()):
+        ns = elm.tag
+        ns = ns[0:ns.rindex("}")+1]
+        name = elm.find("%sname" % ns)
+        summary = elm.find("%ssummary" % ns)
+        if name.text == group or summary.text == group:
+            found = True
+            break
+
+    if not found:
+        return pkglist
+
+    found = False
+    for requires in list(elm):
+        if requires.tag.endswith("requires"):
+            found = True
+            break
+
+    if not found:
+        return pkglist
+
+    for pkg in list(requires):
+        pkgname = pkg.attrib["name"]
+        if pkgname not in pkglist:
+            pkglist.append(pkgname)
+
+    return pkglist
+
+def get_pkglist_in_comps(group, comps):
+    found = False
+    pkglist = []
+    try:
+        root = xmlparse(comps)
+    except SyntaxError:
+        raise SyntaxError("%s syntax error." % comps)
+
+    for elm in root.getiterator("group"):
+        id = elm.find("id")
+        name = elm.find("name")
+        if id.text == group or name.text == group:
+            packagelist = elm.find("packagelist")
+            found = True
+            break
+
+    if not found:
+        return pkglist
+
+    for require in elm.getiterator("packagereq"):
+        if require.tag.endswith("packagereq"):
+            pkgname = require.text
+        if pkgname not in pkglist:
+            pkglist.append(pkgname)
+
+    return pkglist
+
+def is_statically_linked(binary):
+    return ", statically linked, " in runner.outs(['file', binary])
+
+def setup_qemu_emulator(rootdir, arch):
+    # mount binfmt_misc if it doesn't exist
+    if not os.path.exists("/proc/sys/fs/binfmt_misc"):
+        modprobecmd = find_binary_path("modprobe")
+        runner.show([modprobecmd, "binfmt_misc"])
+    if not os.path.exists("/proc/sys/fs/binfmt_misc/register"):
+        mountcmd = find_binary_path("mount")
+        runner.show([mountcmd, "-t", "binfmt_misc", "none", "/proc/sys/fs/binfmt_misc"])
+
+    # qemu_emulator is a special case, we can't use find_binary_path
+    # qemu emulator should be a statically-linked executable file
+    qemu_emulator = "/usr/bin/qemu-arm"
+    if not os.path.exists(qemu_emulator) or not is_statically_linked(qemu_emulator):
+        qemu_emulator = "/usr/bin/qemu-arm-static"
+    if not os.path.exists(qemu_emulator):
+        raise CreatorError("Please install a statically-linked qemu-arm")
+
+    # qemu emulator version check
+    armv7_list = [arch for arch in rpmmisc.archPolicies.keys() if arch.startswith('armv7')]
+    if arch in armv7_list:  # need qemu (>=0.13.0)
+        qemuout = runner.outs([qemu_emulator, "-h"])
+        m = re.search("version\s*([.\d]+)", qemuout)
+        if m:
+            qemu_version = m.group(1)
+            if qemu_version < "0.13":
+                raise CreatorError("Requires %s version >=0.13 for %s" % (qemu_emulator, arch))
+        else:
+            msger.warning("Can't get version info of %s, please make sure it's higher than 0.13.0" % qemu_emulator)
+
+    if not os.path.exists(rootdir + "/usr/bin"):
+        makedirs(rootdir + "/usr/bin")
+    shutil.copy(qemu_emulator, rootdir + "/usr/bin/qemu-arm-static")
+    qemu_emulator = "/usr/bin/qemu-arm-static"
+
+    # disable selinux, selinux will block qemu emulator to run
+    if os.path.exists("/usr/sbin/setenforce"):
+        msger.info('Try to disable selinux')
+        runner.show(["/usr/sbin/setenforce", "0"])
+
+    # unregister it if it has been registered and is a dynamically-linked executable
+    node = "/proc/sys/fs/binfmt_misc/arm"
+    if os.path.exists(node):
+        qemu_unregister_string = "-1\n"
+        fd = open("/proc/sys/fs/binfmt_misc/arm", "w")
+        fd.write(qemu_unregister_string)
+        fd.close()
+
+    # register qemu emulator for interpreting other arch executable file
+    if not os.path.exists(node):
+        qemu_arm_string = ":arm:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfa\\xff\\xff\\xff:%s:\n" % qemu_emulator
+        fd = open("/proc/sys/fs/binfmt_misc/register", "w")
+        fd.write(qemu_arm_string)
+        fd.close()
+
+    return qemu_emulator
+
+def SrcpkgsDownload(pkgs, repometadata, instroot, cachedir):
+    def get_source_repometadata(repometadata):
+        src_repometadata=[]
+        for repo in repometadata:
+            if repo["name"].endswith("-source"):
+                src_repometadata.append(repo)
+        if src_repometadata:
+            return src_repometadata
+        return None
+
+    def get_src_name(srpm):
+        m = SRPM_RE.match(srpm)
+        if m:
+            return m.group(1)
+        return None
+
+    src_repometadata = get_source_repometadata(repometadata)
+
+    if not src_repometadata:
+        msger.warning("No source repo found")
+        return None
+
+    src_pkgs = []
+    lpkgs_dict = {}
+    lpkgs_path = []
+    for repo in src_repometadata:
+        cachepath = "%s/%s/packages/*.src.rpm" %(cachedir, repo["name"])
+        lpkgs_path += glob.glob(cachepath)
+
+    for lpkg in lpkgs_path:
+        lpkg_name = get_src_name(os.path.basename(lpkg))
+        lpkgs_dict[lpkg_name] = lpkg
+    localpkgs = lpkgs_dict.keys()
+
+    cached_count = 0
+    destdir = instroot+'/usr/src/SRPMS'
+    if not os.path.exists(destdir):
+        os.makedirs(destdir)
+
+    srcpkgset = set()
+    for _pkg in pkgs:
+        srcpkg_name = get_source_name(_pkg, repometadata)
+        if not srcpkg_name:
+            continue
+        srcpkgset.add(srcpkg_name)
+
+    for pkg in list(srcpkgset):
+        if pkg in localpkgs:
+            cached_count += 1
+            shutil.copy(lpkgs_dict[pkg], destdir)
+            src_pkgs.append(os.path.basename(lpkgs_dict[pkg]))
+        else:
+            src_pkg = get_package(pkg, src_repometadata, 'src')
+            if src_pkg:
+                shutil.copy(src_pkg, destdir)
+                src_pkgs.append(src_pkg)
+    msger.info("%d source packages gotten from cache" % cached_count)
+
+    return src_pkgs
+
+def strip_end(text, suffix):
+    if not text.endswith(suffix):
+        return text
+    return text[:-len(suffix)]