2 files changed, 320 insertions, 0 deletions

diff --git a/meta/classes/utility-tasks.bbclass b/meta/classes/utility-tasks.bbclass
index 8bec8d463a..64d65c8aed 100644
--- a/meta/classes/utility-tasks.bbclass
+++ b/meta/classes/utility-tasks.bbclass
@@ -95,3 +95,25 @@ do_buildall() {
         :
 }
 
+addtask distro_check after do_distro_check
+do_distro_check[nostamp] = "1"
+python do_distro_check() {
+    """checks if the package is present in other public Linux distros"""
+    import oe.distro_check as dc
+    localdata = bb.data.createCopy(d)
+    bb.data.update_data(localdata)
+
+    tmpdir = bb.data.getVar('TMPDIR', localdata, 1)
+    distro_check_dir = os.path.join(tmpdir, "distro_check")
+    datetime = bb.data.getVar('DATETIME', localdata, 1)
+
+    # if distro packages list data is old then rebuild it 
+    dc.update_distro_data(distro_check_dir, datetime)
+
+    # do the comparison
+    result = dc.compare_in_distro_packages_list(distro_check_dir, d)
+
+    # save the results
+    dc.save_distro_check_result(result, datetime, d)
+}
+
diff --git a/meta/lib/oe/distro_check.py b/meta/lib/oe/distro_check.py
new file mode 100644
index 0000000000..189f5ef20d
--- /dev/null
+++ b/meta/lib/oe/distro_check.py
@@ -0,0 +1,298 @@
+def get_links_from_url(url):
+    "Return all the href links found on the web location"
+
+    import urllib, sgmllib
+    
+    class LinksParser(sgmllib.SGMLParser):
+        def parse(self, s):
+            "Parse the given string 's'."
+            self.feed(s)
+            self.close()
+    
+        def __init__(self, verbose=0):
+            "Initialise an object passing 'verbose' to the superclass."
+            sgmllib.SGMLParser.__init__(self, verbose)
+            self.hyperlinks = []
+    
+        def start_a(self, attributes):
+            "Process a hyperlink and its 'attributes'."
+            for name, value in attributes:
+                if name == "href":
+                    self.hyperlinks.append(value.strip('/'))
+    
+        def get_hyperlinks(self):
+            "Return the list of hyperlinks."
+            return self.hyperlinks
+
+    sock = urllib.urlopen(url)
+    webpage = sock.read()
+    sock.close()
+
+    linksparser = LinksParser()
+    linksparser.parse(webpage)
+    return linksparser.get_hyperlinks()
+
+def find_latest_numeric_release(url):
+    "Find the latest listed numeric release on the given url"
+    max=0
+    maxstr=""
+    for link in get_links_from_url(url):
+        try:
+            release = float(link)
+        except:
+            release = 0
+        if release > max:
+            max = release
+            maxstr = link
+    return maxstr
+
+def is_src_rpm(name):
+    "Check if the link is pointing to a src.rpm file"
+    if name[-8:] == ".src.rpm":
+        return True
+    else:
+        return False
+
+def package_name_from_srpm(srpm):
+    "Strip out the package name from the src.rpm filename"
+    strings = srpm.split('-')
+    package_name = strings[0]
+    for i in range(1, len (strings) - 1):
+        str = strings[i]
+        if not str[0].isdigit():
+            package_name += '-' + str
+    return package_name
+
+def clean_package_list(package_list):
+    "Removes multiple entries of packages and sorts the list"
+    set = {}
+    map(set.__setitem__, package_list, [])
+    return set.keys()
+
+
+def get_latest_released_fedora_source_package_list():
+    "Returns list of all the name os packages in the latest fedora distro"
+    latest = find_latest_numeric_release("http://download.fedora.redhat.com/pub/fedora/linux/releases/")
+
+    url = "http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest
+    links = get_links_from_url(url)
+    url = "http://download.fedora.redhat.com/pub/fedora/linux/updates/%s/SRPMS/" % latest
+    links += get_links_from_url(url)
+
+    srpms = filter(is_src_rpm, links)
+
+    package_names = map(package_name_from_srpm, srpms)
+    package_list=clean_package_list(package_names)
+        
+    return latest, package_list
+
+def get_latest_released_opensuse_source_package_list():
+    "Returns list of all the name os packages in the latest opensuse distro"
+    latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/")
+
+    url = "http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest
+    links = get_links_from_url(url)
+    url = "http://download.opensuse.org/update/%s/rpm/src/" % latest
+    links += get_links_from_url(url)
+    srpms = filter(is_src_rpm, links)
+
+    package_names = map(package_name_from_srpm, srpms)
+    package_list=clean_package_list(package_names)
+    return latest, package_list
+
+def get_latest_released_mandriva_source_package_list():
+    "Returns list of all the name os packages in the latest mandriva distro"
+    latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/")
+    url = "http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest
+    links = get_links_from_url(url)
+    url = "http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest
+    links += get_links_from_url(url)
+
+    srpms = filter(is_src_rpm, links)
+
+    package_names = map(package_name_from_srpm, srpms)
+    package_list=clean_package_list(package_names)
+    return latest, package_list
+
+def find_latest_debian_release(url):
+    "Find the latest listed debian release on the given url"
+
+    releases = []
+    for link in get_links_from_url(url):
+        if link[:6] == "Debian":
+            if ';' not in link:
+                releases.append(link)
+    releases.sort()
+    try:
+        return releases.pop()[6:]
+    except:
+        return "_NotFound_"
+
+def get_debian_style_source_package_list(url):
+    "Return the list of package-names stored in the debian style Sources.gz file"
+    import urllib
+    sock = urllib.urlopen(url)
+    import tempfile
+    tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='poky.', suffix='.tmp', delete=False)
+    tmpfilename=tmpfile.name
+    tmpfile.write(sock.read())
+    sock.close()
+    tmpfile.close()
+    import gzip
+
+    f = gzip.open(tmpfilename)
+    package_names = []
+    for line in f:
+        if line[:9] == "Package: ":
+            package_names.append(line[9:-1]) # Also strip the '\n' at the end
+    os.unlink(tmpfilename)
+
+    return package_names
+
+def get_latest_released_debian_source_package_list():
+    "Returns list of all the name os packages in the latest debian distro"
+    latest = find_latest_debian_release("ftp://ftp.debian.org/debian/dists/")
+    url = "ftp://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" 
+    package_names = get_debian_style_source_package_list(url)
+    url = "ftp://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" 
+    package_names += get_debian_style_source_package_list(url)
+    package_list=clean_package_list(package_names)
+    return latest, package_list
+
+def find_latest_ubuntu_release(url):
+    "Find the latest listed ubuntu release on the given url"
+    url += "?C=M;O=D" # Descending Sort by Last Modified
+    for link in get_links_from_url(url):
+        if link[-8:] == "-updates":
+            return link[:-8]
+    return "_NotFound_"
+
+def get_latest_released_ubuntu_source_package_list():
+    "Returns list of all the name os packages in the latest ubuntu distro"
+    latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/")
+    url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest
+    package_names = get_debian_style_source_package_list(url)
+    url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest
+    package_names += get_debian_style_source_package_list(url)
+    package_list=clean_package_list(package_names)
+    return latest, package_list
+
+def create_distro_packages_list(distro_check_dir):
+    pkglst_dir = os.path.join(distro_check_dir, "package_lists")
+    if not os.path.isdir (pkglst_dir):
+        os.makedirs(pkglst_dir)
+    # first clear old stuff
+    for file in os.listdir(pkglst_dir):
+        os.unlink(os.path.join(pkglst_dir, file))
+ 
+    per_distro_functions = [["Fedora", get_latest_released_fedora_source_package_list],
+                            ["OpenSuSE", get_latest_released_opensuse_source_package_list],
+                            ["Ubuntu", get_latest_released_ubuntu_source_package_list],
+                            ["Debian", get_latest_released_debian_source_package_list],
+                            ["Mandriva", get_latest_released_mandriva_source_package_list]]
+ 
+    from datetime import datetime
+    begin = datetime.now()
+    for distro in per_distro_functions:
+        name = distro[0]
+        release, package_list = distro[1]()
+        bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list)))
+        package_list_file = os.path.join(pkglst_dir, name + "-" + release)
+        f = open(package_list_file, "w+b")
+        for pkg in package_list:
+            f.write(pkg + "\n")
+        f.close()
+    end = datetime.now()
+    delta = end - begin
+    bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds)
+
+def update_distro_data(distro_check_dir, datetime):
+    """
+        If distro packages list data is old then rebuild it.
+        The operations has to be protected by a lock so that
+        only one thread performes it at a time.
+    """
+    if not os.path.isdir (distro_check_dir):
+        try:
+            bb.note ("Making new directory: %s" % distro_check_dir)
+            os.makedirs (distro_check_dir)
+        except OSError:
+            raise Exception('Unable to create directory %s' % (distro_check_dir))
+
+
+    datetime_file = os.path.join(distro_check_dir, "build_datetime")
+    saved_datetime = "_invalid_"
+    import fcntl
+    try:
+        if not os.path.exists(datetime_file):
+            open(datetime_file, 'w+b').close() # touch the file so that the next open won't fail
+
+        f = open(datetime_file, "r+b")
+        fcntl.lockf(f, fcntl.LOCK_EX)
+        saved_datetime = f.read()
+        if saved_datetime != datetime:
+            bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime))
+            bb.note("Regenerating distro package lists")
+            create_distro_packages_list(distro_check_dir)
+            f.seek(0)
+            f.write(datetime)
+
+    except OSError:
+        raise Exception('Unable to read/write this file: %s' % (datetime_file))
+    finally:
+        fcntl.lockf(f, fcntl.LOCK_UN)
+        f.close()
+ 
+def compare_in_distro_packages_list(distro_check_dir, d):
+    if not os.path.isdir(distro_check_dir):
+        raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed")
+        
+    pkglst_dir = os.path.join(distro_check_dir, "package_lists")
+    matching_distros = []
+    recipe_name = bb.data.getVar('PN', d, True)
+    tmp = bb.data.getVar('DISTRO_PN_ALIAS', d, True)
+    distro_pn_aliases = {}
+    if tmp:
+        list = tmp.split(';')
+        for str in list:
+            (dist, pn_alias) = str.split('=')
+            distro_pn_aliases[dist.strip().lower()] = pn_alias.strip()
+ 
+    for file in os.listdir(pkglst_dir):
+        (distro, distro_release) = file.split("-")
+        f = open(os.path.join(pkglst_dir, file), "rb")
+        for pkg in f:
+            if distro.lower() in distro_pn_aliases:
+                pn = distro_pn_aliases[distro.lower()]
+            else:
+                pn = recipe_name
+            if pn == pkg[:-1]: # strip the \n at the end
+                matching_distros.append(distro)
+                f.close()
+                break
+        f.close()
+    return matching_distros
+
+def save_distro_check_result(result, datetime, d):
+    pn = bb.data.getVar('PN', d, True)
+    logdir = bb.data.getVar('LOG_DIR', d, True)
+    if not logdir:
+        bb.error("LOG_DIR variable is not defined, can't write the distro_check results")
+        return
+    if not os.path.isdir(logdir):
+        os.makedirs(logdir)
+    result_file = os.path.join(logdir, "distro_check-" + datetime + ".results")
+    line = pn + " : "
+    for i in result:
+        line = line + i + ", "
+    if result:
+        line = line[:-2] # Take out the comma at the end of line
+    if not os.path.exists(result_file):
+        open(result_file, 'w+b').close() # touch the file so that the next open won't fail
+    f = open(result_file, "a+b")
+    import fcntl
+    fcntl.lockf(f, fcntl.LOCK_EX)
+    f.seek(0, os.SEEK_END) # seek to the end of file
+    f.write(line + "\n")
+    fcntl.lockf(f, fcntl.LOCK_UN)
+    f.close()