def get_links_from_url(url): "Return all the href links found on the web location" import urllib, sgmllib class LinksParser(sgmllib.SGMLParser): def parse(self, s): "Parse the given string 's'." self.feed(s) self.close() def __init__(self, verbose=0): "Initialise an object passing 'verbose' to the superclass." sgmllib.SGMLParser.__init__(self, verbose) self.hyperlinks = [] def start_a(self, attributes): "Process a hyperlink and its 'attributes'." for name, value in attributes: if name == "href": self.hyperlinks.append(value.strip('/')) def get_hyperlinks(self): "Return the list of hyperlinks." return self.hyperlinks sock = urllib.urlopen(url) webpage = sock.read() sock.close() linksparser = LinksParser() linksparser.parse(webpage) return linksparser.get_hyperlinks() def find_latest_numeric_release(url): "Find the latest listed numeric release on the given url" max=0 maxstr="" for link in get_links_from_url(url): try: release = float(link) except: release = 0 if release > max: max = release maxstr = link return maxstr def is_src_rpm(name): "Check if the link is pointing to a src.rpm file" if name[-8:] == ".src.rpm": return True else: return False def package_name_from_srpm(srpm): "Strip out the package name from the src.rpm filename" strings = srpm.split('-') package_name = strings[0] for i in range(1, len (strings) - 1): str = strings[i] if not str[0].isdigit(): package_name += '-' + str return package_name def clean_package_list(package_list): "Removes multiple entries of packages and sorts the list" set = {} map(set.__setitem__, package_list, []) return set.keys() def get_latest_released_fedora_source_package_list(): "Returns list of all the name os packages in the latest fedora distro" latest = find_latest_numeric_release("http://download.fedora.redhat.com/pub/fedora/linux/releases/") url = "http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest links = get_links_from_url(url) url = "http://download.fedora.redhat.com/pub/fedora/linux/updates/%s/SRPMS/" % latest links += get_links_from_url(url) srpms = filter(is_src_rpm, links) package_names = map(package_name_from_srpm, srpms) package_list=clean_package_list(package_names) return latest, package_list def get_latest_released_opensuse_source_package_list(): "Returns list of all the name os packages in the latest opensuse distro" latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/") url = "http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest links = get_links_from_url(url) url = "http://download.opensuse.org/update/%s/rpm/src/" % latest links += get_links_from_url(url) srpms = filter(is_src_rpm, links) package_names = map(package_name_from_srpm, srpms) package_list=clean_package_list(package_names) return latest, package_list def get_latest_released_mandriva_source_package_list(): "Returns list of all the name os packages in the latest mandriva distro" latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/") url = "http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest links = get_links_from_url(url) url = "http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest links += get_links_from_url(url) srpms = filter(is_src_rpm, links) package_names = map(package_name_from_srpm, srpms) package_list=clean_package_list(package_names) return latest, package_list def find_latest_debian_release(url): "Find the latest listed debian release on the given url" releases = [] for link in get_links_from_url(url): if link[:6] == "Debian": if ';' not in link: releases.append(link) releases.sort() try: return releases.pop()[6:] except: return "_NotFound_" def get_debian_style_source_package_list(url): "Return the list of package-names stored in the debian style Sources.gz file" import urllib sock = urllib.urlopen(url) import tempfile tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='poky.', suffix='.tmp', delete=False) tmpfilename=tmpfile.name tmpfile.write(sock.read()) sock.close() tmpfile.close() import gzip f = gzip.open(tmpfilename) package_names = [] for line in f: if line[:9] == "Package: ": package_names.append(line[9:-1]) # Also strip the '\n' at the end os.unlink(tmpfilename) return package_names def get_latest_released_debian_source_package_list(): "Returns list of all the name os packages in the latest debian distro" latest = find_latest_debian_release("ftp://ftp.debian.org/debian/dists/") url = "ftp://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" package_names = get_debian_style_source_package_list(url) url = "ftp://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" package_names += get_debian_style_source_package_list(url) package_list=clean_package_list(package_names) return latest, package_list def find_latest_ubuntu_release(url): "Find the latest listed ubuntu release on the given url" url += "?C=M;O=D" # Descending Sort by Last Modified for link in get_links_from_url(url): if link[-8:] == "-updates": return link[:-8] return "_NotFound_" def get_latest_released_ubuntu_source_package_list(): "Returns list of all the name os packages in the latest ubuntu distro" latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/") url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest package_names = get_debian_style_source_package_list(url) url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest package_names += get_debian_style_source_package_list(url) package_list=clean_package_list(package_names) return latest, package_list def create_distro_packages_list(distro_check_dir): pkglst_dir = os.path.join(distro_check_dir, "package_lists") if not os.path.isdir (pkglst_dir): os.makedirs(pkglst_dir) # first clear old stuff for file in os.listdir(pkglst_dir): os.unlink(os.path.join(pkglst_dir, file)) per_distro_functions = [["Fedora", get_latest_released_fedora_source_package_list], ["OpenSuSE", get_latest_released_opensuse_source_package_list], ["Ubuntu", get_latest_released_ubuntu_source_package_list], ["Debian", get_latest_released_debian_source_package_list], ["Mandriva", get_latest_released_mandriva_source_package_list]] from datetime import datetime begin = datetime.now() for distro in per_distro_functions: name = distro[0] release, package_list = distro[1]() bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list))) package_list_file = os.path.join(pkglst_dir, name + "-" + release) f = open(package_list_file, "w+b") for pkg in package_list: f.write(pkg + "\n") f.close() end = datetime.now() delta = end - begin bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds) def update_distro_data(distro_check_dir, datetime): """ If distro packages list data is old then rebuild it. The operations has to be protected by a lock so that only one thread performes it at a time. """ if not os.path.isdir (distro_check_dir): try: bb.note ("Making new directory: %s" % distro_check_dir) os.makedirs (distro_check_dir) except OSError: raise Exception('Unable to create directory %s' % (distro_check_dir)) datetime_file = os.path.join(distro_check_dir, "build_datetime") saved_datetime = "_invalid_" import fcntl try: if not os.path.exists(datetime_file): open(datetime_file, 'w+b').close() # touch the file so that the next open won't fail f = open(datetime_file, "r+b") fcntl.lockf(f, fcntl.LOCK_EX) saved_datetime = f.read() if saved_datetime != datetime: bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime)) bb.note("Regenerating distro package lists") create_distro_packages_list(distro_check_dir) f.seek(0) f.write(datetime) except OSError: raise Exception('Unable to read/write this file: %s' % (datetime_file)) finally: fcntl.lockf(f, fcntl.LOCK_UN) f.close() def compare_in_distro_packages_list(distro_check_dir, d): if not os.path.isdir(distro_check_dir): raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed") pkglst_dir = os.path.join(distro_check_dir, "package_lists") matching_distros = [] recipe_name = bb.data.getVar('PN', d, True) tmp = bb.data.getVar('DISTRO_PN_ALIAS', d, True) distro_pn_aliases = {} if tmp: list = tmp.split(';') for str in list: (dist, pn_alias) = str.split('=') distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() for file in os.listdir(pkglst_dir): (distro, distro_release) = file.split("-") f = open(os.path.join(pkglst_dir, file), "rb") for pkg in f: if distro.lower() in distro_pn_aliases: pn = distro_pn_aliases[distro.lower()] else: pn = recipe_name if pn == pkg[:-1]: # strip the \n at the end matching_distros.append(distro) f.close() break f.close() return matching_distros def save_distro_check_result(result, datetime, d): pn = bb.data.getVar('PN', d, True) logdir = bb.data.getVar('LOG_DIR', d, True) if not logdir: bb.error("LOG_DIR variable is not defined, can't write the distro_check results") return if not os.path.isdir(logdir): os.makedirs(logdir) result_file = os.path.join(logdir, "distro_check-" + datetime + ".results") line = pn + " : " for i in result: line = line + i + ", " if result: line = line[:-2] # Take out the comma at the end of line if not os.path.exists(result_file): open(result_file, 'w+b').close() # touch the file so that the next open won't fail f = open(result_file, "a+b") import fcntl fcntl.lockf(f, fcntl.LOCK_EX) f.seek(0, os.SEEK_END) # seek to the end of file f.write(line + "\n") fcntl.lockf(f, fcntl.LOCK_UN) f.close()