From c527fd1f14c27855a37f2e8ac5346ce8d940ced2 Mon Sep 17 00:00:00 2001 From: Tudor Florea Date: Thu, 16 Oct 2014 03:05:19 +0200 Subject: initial commit for Enea Linux 4.0-140929 Migrated from the internal git server on the daisy-enea-point-release branch Signed-off-by: Tudor Florea --- meta/classes/distrodata.bbclass | 916 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 916 insertions(+) create mode 100644 meta/classes/distrodata.bbclass (limited to 'meta/classes/distrodata.bbclass') diff --git a/meta/classes/distrodata.bbclass b/meta/classes/distrodata.bbclass new file mode 100644 index 0000000000..b47358b059 --- /dev/null +++ b/meta/classes/distrodata.bbclass @@ -0,0 +1,916 @@ +include conf/distro/include/package_regex.inc +addhandler distro_eventhandler +distro_eventhandler[eventmask] = "bb.event.BuildStarted" +python distro_eventhandler() { + import oe.distro_check as dc + logfile = dc.create_log_file(e.data, "distrodata.csv") + lf = bb.utils.lockfile("%s.lock" % logfile) + f = open(logfile, "a") + f.write("Package,Description,Owner,License,VerMatch,Version,Upsteam,Reason,Recipe Status,Distro 1,Distro 2,Distro 3\n") + f.close() + bb.utils.unlockfile(lf) + + return +} + +addtask distrodata_np +do_distrodata_np[nostamp] = "1" +python do_distrodata_np() { + localdata = bb.data.createCopy(d) + pn = d.getVar("PN", True) + bb.note("Package Name: %s" % pn) + + import oe.distro_check as dist_check + tmpdir = d.getVar('TMPDIR', True) + distro_check_dir = os.path.join(tmpdir, "distro_check") + datetime = localdata.getVar('DATETIME', True) + dist_check.update_distro_data(distro_check_dir, datetime) + + if pn.find("-native") != -1: + pnstripped = pn.split("-native") + bb.note("Native Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) + bb.data.update_data(localdata) + + if pn.find("-cross") != -1: + pnstripped = pn.split("-cross") + bb.note("cross Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) + bb.data.update_data(localdata) + + if pn.find("-crosssdk") != -1: + pnstripped = pn.split("-crosssdk") + bb.note("cross Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) + bb.data.update_data(localdata) + + if pn.startswith("nativesdk-"): + pnstripped = pn.replace("nativesdk-", "") + bb.note("NativeSDK Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped + ":" + d.getVar('OVERRIDES', True)) + bb.data.update_data(localdata) + + + if pn.find("-initial") != -1: + pnstripped = pn.split("-initial") + bb.note("initial Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) + bb.data.update_data(localdata) + + """generate package information from .bb file""" + pname = localdata.getVar('PN', True) + pcurver = localdata.getVar('PV', True) + pdesc = localdata.getVar('DESCRIPTION', True) + if pdesc is not None: + pdesc = pdesc.replace(',','') + pdesc = pdesc.replace('\n','') + + pgrp = localdata.getVar('SECTION', True) + plicense = localdata.getVar('LICENSE', True).replace(',','_') + + rstatus = localdata.getVar('RECIPE_COLOR', True) + if rstatus is not None: + rstatus = rstatus.replace(',','') + + pupver = localdata.getVar('RECIPE_UPSTREAM_VERSION', True) + if pcurver == pupver: + vermatch="1" + else: + vermatch="0" + noupdate_reason = localdata.getVar('RECIPE_NO_UPDATE_REASON', True) + if noupdate_reason is None: + noupdate="0" + else: + noupdate="1" + noupdate_reason = noupdate_reason.replace(',','') + + maintainer = localdata.getVar('RECIPE_MAINTAINER', True) + rlrd = localdata.getVar('RECIPE_UPSTREAM_DATE', True) + result = dist_check.compare_in_distro_packages_list(distro_check_dir, localdata) + + bb.note("DISTRO: %s,%s,%s,%s,%s,%s,%s,%s,%s\n" % \ + (pname, pdesc, maintainer, plicense, vermatch, pcurver, pupver, noupdate_reason, rstatus)) + line = pn + for i in result: + line = line + "," + i + bb.note("%s\n" % line) +} + +addtask distrodata +do_distrodata[nostamp] = "1" +python do_distrodata() { + logpath = d.getVar('LOG_DIR', True) + bb.utils.mkdirhier(logpath) + logfile = os.path.join(logpath, "distrodata.csv") + + import oe.distro_check as dist_check + localdata = bb.data.createCopy(d) + tmpdir = d.getVar('TMPDIR', True) + distro_check_dir = os.path.join(tmpdir, "distro_check") + datetime = localdata.getVar('DATETIME', True) + dist_check.update_distro_data(distro_check_dir, datetime) + + pn = d.getVar("PN", True) + bb.note("Package Name: %s" % pn) + + if pn.find("-native") != -1: + pnstripped = pn.split("-native") + bb.note("Native Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) + bb.data.update_data(localdata) + + if pn.startswith("nativesdk-"): + pnstripped = pn.replace("nativesdk-", "") + bb.note("NativeSDK Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped + ":" + d.getVar('OVERRIDES', True)) + bb.data.update_data(localdata) + + if pn.find("-cross") != -1: + pnstripped = pn.split("-cross") + bb.note("cross Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) + bb.data.update_data(localdata) + + if pn.find("-crosssdk") != -1: + pnstripped = pn.split("-crosssdk") + bb.note("cross Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) + bb.data.update_data(localdata) + + if pn.find("-initial") != -1: + pnstripped = pn.split("-initial") + bb.note("initial Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) + bb.data.update_data(localdata) + + """generate package information from .bb file""" + pname = localdata.getVar('PN', True) + pcurver = localdata.getVar('PV', True) + pdesc = localdata.getVar('DESCRIPTION', True) + if pdesc is not None: + pdesc = pdesc.replace(',','') + pdesc = pdesc.replace('\n','') + + pgrp = localdata.getVar('SECTION', True) + plicense = localdata.getVar('LICENSE', True).replace(',','_') + + rstatus = localdata.getVar('RECIPE_COLOR', True) + if rstatus is not None: + rstatus = rstatus.replace(',','') + + pupver = localdata.getVar('RECIPE_UPSTREAM_VERSION', True) + if pcurver == pupver: + vermatch="1" + else: + vermatch="0" + + noupdate_reason = localdata.getVar('RECIPE_NO_UPDATE_REASON', True) + if noupdate_reason is None: + noupdate="0" + else: + noupdate="1" + noupdate_reason = noupdate_reason.replace(',','') + + maintainer = localdata.getVar('RECIPE_MAINTAINER', True) + rlrd = localdata.getVar('RECIPE_UPSTREAM_DATE', True) + # do the comparison + result = dist_check.compare_in_distro_packages_list(distro_check_dir, localdata) + + lf = bb.utils.lockfile("%s.lock" % logfile) + f = open(logfile, "a") + f.write("%s,%s,%s,%s,%s,%s,%s,%s,%s" % \ + (pname, pdesc, maintainer, plicense, vermatch, pcurver, pupver, noupdate_reason, rstatus)) + line = "" + for i in result: + line = line + "," + i + f.write(line + "\n") + f.close() + bb.utils.unlockfile(lf) +} + +addtask distrodataall after do_distrodata +do_distrodataall[recrdeptask] = "do_distrodataall do_distrodata" +do_distrodataall[recideptask] = "do_${BB_DEFAULT_TASK}" +do_distrodataall[nostamp] = "1" +do_distrodataall() { + : +} + +addhandler checkpkg_eventhandler +checkpkg_eventhandler[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted" +python checkpkg_eventhandler() { + def parse_csv_file(filename): + package_dict = {} + fd = open(filename, "r") + lines = fd.read().rsplit("\n") + fd.close() + + first_line = '' + index = 0 + for line in lines: + #Skip the first line + if index == 0: + first_line = line + index += 1 + continue + elif line == '': + continue + index += 1 + package_name = line.rsplit("\t")[0] + if '-native' in package_name or 'nativesdk-' in package_name: + original_name = package_name.rsplit('-native')[0] + if original_name == '': + original_name = package_name.rsplit('nativesdk-')[0] + if original_name in package_dict: + continue + else: + package_dict[package_name] = line + else: + new_name = package_name + "-native" + if not(new_name in package_dict): + new_name = 'nativesdk-' + package_name + if new_name in package_dict: + del package_dict[new_name] + package_dict[package_name] = line + + fd = open(filename, "w") + fd.write("%s\n"%first_line) + for el in package_dict: + fd.write(package_dict[el] + "\n") + fd.close() + + del package_dict + + if bb.event.getName(e) == "BuildStarted": + import oe.distro_check as dc + logfile = dc.create_log_file(e.data, "checkpkg.csv") + + lf = bb.utils.lockfile("%s.lock" % logfile) + f = open(logfile, "a") + f.write("Package\tVersion\tUpver\tLicense\tSection\tHome\tRelease\tDepends\tBugTracker\tPE\tDescription\tStatus\tTracking\tURI\tMAINTAINER\tNoUpReason\n") + f.close() + bb.utils.unlockfile(lf) + elif bb.event.getName(e) == "BuildCompleted": + import os + filename = "tmp/log/checkpkg.csv" + if os.path.isfile(filename): + lf = bb.utils.lockfile("%s.lock"%filename) + parse_csv_file(filename) + bb.utils.unlockfile(lf) + return +} + +addtask checkpkg +do_checkpkg[nostamp] = "1" +python do_checkpkg() { + localdata = bb.data.createCopy(d) + import re + import tempfile + import subprocess + + """ + sanity check to ensure same name and type. Match as many patterns as possible + such as: + gnome-common-2.20.0.tar.gz (most common format) + gtk+-2.90.1.tar.gz + xf86-input-synaptics-12.6.9.tar.gz + dri2proto-2.3.tar.gz + blktool_4.orig.tar.gz + libid3tag-0.15.1b.tar.gz + unzip552.tar.gz + icu4c-3_6-src.tgz + genext2fs_1.3.orig.tar.gz + gst-fluendo-mp3 + """ + prefix1 = "[a-zA-Z][a-zA-Z0-9]*([\-_][a-zA-Z]\w+)*\+?[\-_]" # match most patterns which uses "-" as separator to version digits + prefix2 = "[a-zA-Z]+" # a loose pattern such as for unzip552.tar.gz + prefix3 = "[0-9]+[\-]?[a-zA-Z]+" # a loose pattern such as for 80325-quicky-0.4.tar.gz + prefix = "(%s|%s|%s)" % (prefix1, prefix2, prefix3) + ver_regex = "(([A-Z]*\d+[a-zA-Z]*[\.\-_]*)+)"#"((\d+[\.\-_[a-z]])+)" + # src.rpm extension was added only for rpm package. Can be removed if the rpm + # packaged will always be considered as having to be manually upgraded + suffix = "(tar\.gz|tgz|tar\.bz2|tar\.lz4|zip|xz|rpm|bz2|lz4|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)" + + suffixtuple = ("tar.gz", "tgz", "zip", "tar.bz2", "tar.xz", "tar.lz4", "bz2", "lz4", "orig.tar.gz", "src.tar.gz", "src.rpm", "src.tgz", "svnr\d+.tar.bz2", "stable.tar.gz", "src.rpm") + sinterstr = "(?P%s?)v?(?P%s)(\-source)?" % (prefix, ver_regex) + sdirstr = "(?P%s)\.?v?(?P%s)(\-source)?[\.\-](?P%s$)" % (prefix, ver_regex, suffix) + + def parse_inter(s): + m = re.search(sinterstr, s) + if not m: + return None + else: + return (m.group('name'), m.group('ver'), "") + + def parse_dir(s): + m = re.search(sdirstr, s) + if not m: + return None + else: + return (m.group('name'), m.group('ver'), m.group('type')) + + def modelate_version(version): + if version[0] in ['.', '-']: + if version[1].isdigit(): + version = version[1] + version[0] + version[2:len(version)] + else: + version = version[1:len(version)] + + version = re.sub('\-', '.', version) + version = re.sub('_', '.', version) + version = re.sub('(rc)+', '.-1.', version) + version = re.sub('(alpha)+', '.-3.', version) + version = re.sub('(beta)+', '.-2.', version) + if version[0] == 'v': + version = version[1:len(version)] + return version + + """ + Check whether 'new' is newer than 'old' version. We use existing vercmp() for the + purpose. PE is cleared in comparison as it's not for build, and PV is cleared too + for simplicity as it's somehow difficult to get from various upstream format + """ + def __vercmp(old, new): + (on, ov, ot) = old + (en, ev, et) = new + if on != en or (et and et not in suffixtuple): + return False + ov = modelate_version(ov) + ev = modelate_version(ev) + + result = bb.utils.vercmp(("0", ov, ""), ("0", ev, "")) + if result < 0: + return True + else: + return False + + """ + wrapper for fetch upstream directory info + 'url' - upstream link customized by regular expression + 'd' - database + 'tmpf' - tmpfile for fetcher output + We don't want to exit whole build due to one recipe error. So handle all exceptions + gracefully w/o leaking to outer. + """ + def internal_fetch_wget(url, d, tmpf): + status = "ErrFetchUnknown" + """ + Clear internal url cache as it's a temporary check. Not doing so will have + bitbake check url multiple times when looping through a single url + """ + fn = d.getVar('FILE', True) + bb.fetch2.urldata_cache[fn] = {} + + """ + To avoid impacting bitbake build engine, this trick is required for reusing bitbake + interfaces. bb.fetch.go() is not appliable as it checks downloaded content in ${DL_DIR} + while we don't want to pollute that place. So bb.fetch2.checkstatus() is borrowed here + which is designed for check purpose but we override check command for our own purpose + """ + ld = bb.data.createCopy(d) + d.setVar('CHECKCOMMAND_wget', "/usr/bin/env wget -t 1 --passive-ftp -O %s --user-agent=\"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12\" '${URI}'" \ + % tmpf.name) + bb.data.update_data(ld) + + try: + fetcher = bb.fetch2.Fetch([url], ld) + fetcher.checkstatus() + status = "SUCC" + except bb.fetch2.BBFetchException, e: + status = "ErrFetch" + + return status + + """ + Check on middle version directory such as "2.4/" in "http://xxx/2.4/pkg-2.4.1.tar.gz", + 'url' - upstream link customized by regular expression + 'd' - database + 'curver' - current version + Return new version if success, or else error in "Errxxxx" style + """ + def check_new_dir(url, curver, d): + pn = d.getVar('PN', True) + f = tempfile.NamedTemporaryFile(delete=False, prefix="%s-1-" % pn) + status = internal_fetch_wget(url, d, f) + fhtml = f.read() + if status == "SUCC" and len(fhtml): + newver = parse_inter(curver) + + """ + match "*4.1/">*4.1/ where '*' matches chars + N.B. add package name, only match for digits + """ + regex = d.getVar('REGEX', True) + if regex == '': + regex = "^%s" %prefix + m = re.search("^%s" % regex, curver) + if m: + s = "%s[^\d\"]*?(\d+[\.\-_])+\d+/?" % m.group() + else: + s = "(\d+[\.\-_])+\d+/?" + + searchstr = "[hH][rR][eE][fF]=\"%s\">" % s + + reg = re.compile(searchstr) + valid = 0 + for line in fhtml.split("\n"): + if line.find(curver) >= 0: + valid = 1 + m = reg.search(line) + if m: + ver = m.group().split("\"")[1] + ver = ver.strip("/") + ver = parse_inter(ver) + if ver and __vercmp(newver, ver) == True: + newver = ver + + """Expect a match for curver in directory list, or else it indicates unknown format""" + if not valid: + status = "ErrParseInterDir" + else: + """rejoin the path name""" + status = newver[0] + newver[1] + elif not len(fhtml): + status = "ErrHostNoDir" + + f.close() + if status != "ErrHostNoDir" and re.match("Err", status): + logpath = d.getVar('LOG_DIR', True) + subprocess.call("cp %s %s/" % (f.name, logpath), shell=True) + os.unlink(f.name) + return status + + """ + Check on the last directory to search '2.4.1' in "http://xxx/2.4/pkg-2.4.1.tar.gz", + 'url' - upstream link customized by regular expression + 'd' - database + 'curname' - current package name + Return new version if success, or else error in "Errxxxx" style + """ + def check_new_version(url, curname, d): + """possible to have no version in pkg name, such as spectrum-fw""" + if not re.search("\d+", curname): + return pcurver + pn = d.getVar('PN', True) + newver_regex = d.getVar('REGEX', True) + f = tempfile.NamedTemporaryFile(delete=False, prefix="%s-2-" % pn) + status = internal_fetch_wget(url, d, f) + fhtml = f.read() + + if status == "SUCC" and len(fhtml): + newver = parse_dir(curname) + + if not newver_regex: + """this is the default matching pattern, if recipe does not """ + """provide a regex expression """ + """match "{PN}-5.21.1.tar.gz">{PN}-5.21.1.tar.gz """ + pn1 = re.search("^%s" % prefix, curname).group() + s = "[^\"]*%s[^\d\"]*?(\d+[\.\-_])+[^\"]*" % pn1 + searchstr = "[hH][rR][eE][fF]=\"%s\".*[>\"]" % s + reg = searchstr + else: + reg = newver_regex + valid = 0 + count = 0 + for line in fhtml.split("\n"): + if pn == 'kconfig-frontends': + m = re.findall(reg, line) + if m: + valid = 1 + for match in m: + (on, ov, oe) = newver + ver = (on, match[0], oe) + if ver and __vercmp(newver, ver) == True: + newver = ver + continue + count += 1 + m = re.search(reg, line) + if m: + valid = 1 + if not newver_regex: + ver = m.group().split("\"")[1].split("/")[-1] + if ver == "download": + ver = m.group().split("\"")[1].split("/")[-2] + ver = parse_dir(ver) + else: + """ we cheat a little here, but we assume that the + regular expression in the recipe will extract exacly + the version """ + (on, ov, oe) = newver + ver = (on, m.group('pver'), oe) + if ver and __vercmp(newver, ver) == True: + newver = ver + """Expect a match for curver in directory list, or else it indicates unknown format""" + if not valid: + status = "ErrParseDir" + else: + """newver still contains a full package name string""" + status = re.sub('_', '.', newver[1]) + elif not len(fhtml): + status = "ErrHostNoDir" + + f.close() + """if host hasn't directory information, no need to save tmp file""" + if status != "ErrHostNoDir" and re.match("Err", status): + logpath = d.getVar('LOG_DIR', True) + subprocess.call("cp %s %s/" % (f.name, logpath), shell=True) + os.unlink(f.name) + return status + + """first check whether a uri is provided""" + src_uri = d.getVar('SRC_URI', True) + if not src_uri: + return + + """initialize log files.""" + logpath = d.getVar('LOG_DIR', True) + bb.utils.mkdirhier(logpath) + logfile = os.path.join(logpath, "checkpkg.csv") + + """generate package information from .bb file""" + pname = d.getVar('PN', True) + + if pname.find("-native") != -1: + if d.getVar('BBCLASSEXTEND', True): + return + pnstripped = pname.split("-native") + bb.note("Native Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) + bb.data.update_data(localdata) + + if pname.startswith("nativesdk-"): + if d.getVar('BBCLASSEXTEND', True): + return + pnstripped = pname.replace("nativesdk-", "") + bb.note("NativeSDK Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped + ":" + d.getVar('OVERRIDES', True)) + bb.data.update_data(localdata) + + if pname.find("-cross") != -1: + pnstripped = pname.split("-cross") + bb.note("cross Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) + bb.data.update_data(localdata) + + if pname.find("-initial") != -1: + pnstripped = pname.split("-initial") + bb.note("initial Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) + bb.data.update_data(localdata) + + chk_uri = d.getVar('REGEX_URI', True) + if not chk_uri: + chk_uri = src_uri + pdesc = localdata.getVar('DESCRIPTION', True) + pgrp = localdata.getVar('SECTION', True) + if localdata.getVar('PRSPV', True): + pversion = localdata.getVar('PRSPV', True) + else: + pversion = localdata.getVar('PV', True) + plicense = localdata.getVar('LICENSE', True) + psection = localdata.getVar('SECTION', True) + phome = localdata.getVar('HOMEPAGE', True) + prelease = localdata.getVar('PR', True) + pdepends = localdata.getVar('DEPENDS', True) + pbugtracker = localdata.getVar('BUGTRACKER', True) + ppe = localdata.getVar('PE', True) + psrcuri = localdata.getVar('SRC_URI', True) + maintainer = localdata.getVar('RECIPE_MAINTAINER', True) + + found = 0 + for uri in src_uri.split(): + m = re.compile('(?P[^:]*)').match(uri) + if not m: + raise MalformedUrl(uri) + elif m.group('type') in ('http', 'https', 'ftp', 'cvs', 'svn', 'git'): + found = 1 + pproto = m.group('type') + break + if not found: + pproto = "file" + pupver = "N/A" + pstatus = "ErrUnknown" + + (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(uri) + if type in ['http', 'https', 'ftp']: + if d.getVar('PRSPV', True): + pcurver = d.getVar('PRSPV', True) + else: + pcurver = d.getVar('PV', True) + else: + if d.getVar('PRSPV', True): + pcurver = d.getVar('PRSPV', True) + else: + pcurver = d.getVar("SRCREV", True) + + + if type in ['http', 'https', 'ftp']: + newver = pcurver + altpath = path + dirver = "-" + curname = "-" + + """ + match version number amid the path, such as "5.7" in: + http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz + N.B. how about sth. like "../5.7/5.8/..."? Not find such example so far :-P + """ + m = re.search(r"[^/]*(\d+\.)+\d+([\-_]r\d+)*/", path) + if m: + altpath = path.split(m.group())[0] + dirver = m.group().strip("/") + + """use new path and remove param. for wget only param is md5sum""" + alturi = bb.fetch.encodeurl([type, host, altpath, user, pswd, {}]) + my_uri = d.getVar('REGEX_URI', True) + if my_uri: + if d.getVar('PRSPV', True): + newver = d.getVar('PRSPV', True) + else: + newver = d.getVar('PV', True) + else: + newver = check_new_dir(alturi, dirver, d) + altpath = path + if not re.match("Err", newver) and dirver != newver: + altpath = altpath.replace(dirver, newver, True) + # For folder in folder cases - try to enter the folder again and then try parsing + """Now try to acquire all remote files in current directory""" + if not re.match("Err", newver): + curname = altpath.split("/")[-1] + + """get remote name by skipping pacakge name""" + m = re.search(r"/.*/", altpath) + if not m: + altpath = "/" + else: + altpath = m.group() + + chk_uri = d.getVar('REGEX_URI', True) + if not chk_uri: + alturi = bb.fetch.encodeurl([type, host, altpath, user, pswd, {}]) + else: + alturi = chk_uri + newver = check_new_version(alturi, curname, d) + while(newver == "ErrHostNoDir"): + if alturi == "/download": + break + else: + alturi = "/".join(alturi.split("/")[0:-2]) + "/download" + newver = check_new_version(alturi, curname, d) + if not re.match("Err", newver): + pupver = newver + if pupver != pcurver: + pstatus = "UPDATE" + else: + pstatus = "MATCH" + + if re.match("Err", newver): + pstatus = newver + ":" + altpath + ":" + dirver + ":" + curname + elif type == 'git': + if user: + gituser = user + '@' + else: + gituser = "" + + if 'protocol' in parm: + gitproto = parm['protocol'] + else: + gitproto = "git" + + # Get all tags and HEAD + if d.getVar('GIT_REGEX', True): + gitcmd = "git ls-remote %s://%s%s%s %s 2>&1" % (gitproto, gituser, host, path, d.getVar('GIT_REGEX', True)) + else: + gitcmd = "git ls-remote %s://%s%s%s *tag* 2>&1" % (gitproto, gituser, host, path) + gitcmd2 = "git ls-remote %s://%s%s%s HEAD 2>&1" % (gitproto, gituser, host, path) + + tmp = os.popen(gitcmd).read() + if 'unable to connect' in tmp: + tmp = None + tmp2 = os.popen(gitcmd2).read() + if 'unable to connect' in tmp2: + tmp2 = None + #This is for those repos have tag like: refs/tags/1.2.2 + phash = pversion.rsplit("+")[-1] + if tmp: + tmpline = tmp.split("\n") + verflag = 0 + pupver = pversion + for line in tmpline: + if len(line)==0: + break; + puptag = line.split("/")[-1] + upstr_regex = d.getVar('REGEX', True) + if upstr_regex: + puptag = re.search(upstr_regex, puptag) + else: + puptag = re.search("(?P([0-9][\.|_]?)+)", puptag) + if puptag == None: + continue + puptag = puptag.group('pver') + puptag = re.sub("_",".",puptag) + plocaltag = pupver.split("+git")[0] + if "git" in plocaltag: + plocaltag = plocaltag.split("-")[0] + result = bb.utils.vercmp(("0", puptag, ""), ("0", plocaltag, "")) + + if result > 0: + verflag = 1 + pupver = puptag + elif verflag == 0 : + pupver = plocaltag + #This is for those no tag repo + elif tmp2: + pupver = pversion.rsplit("+")[0] + phash = pupver + else: + pstatus = "ErrGitAccess" + if not ('ErrGitAccess' in pstatus): + + latest_head = tmp2.rsplit("\t")[0][:7] + tmp3 = re.search('(?P(\d+[\.-]?)+)(?P(\+git[r|\-|]?)AUTOINC\+)(?P([\w|_]+))', pversion) + tmp4 = re.search('(?P(\d+[\.-]?)+)(?P(\+git[r|\-|]?)AUTOINC\+)(?P([\w|_]+))', pupver) + if not tmp4: + tmp4 = re.search('(?P(\d+[\.-]?)+)', pupver) + + if tmp3: + # Get status of the package - MATCH/UPDATE + result = bb.utils.vercmp(("0", tmp3.group('git_ver'), ""), ("0",tmp3.group('git_ver') , "")) + # Get the latest tag + pstatus = 'MATCH' + if result < 0: + latest_pv = tmp3.group('git_ver') + else: + latest_pv = pupver + if not(tmp3.group('head_md5')[:7] in latest_head) or not(latest_head in tmp3.group('head_md5')[:7]): + pstatus = 'UPDATE' + + git_prefix = tmp3.group('git_prefix') + pupver = latest_pv + tmp3.group('git_prefix') + latest_head + else: + if not tmp3: + bb.plain("#DEBUG# Package %s: current version (%s) doesn't match the usual pattern" %(pname, pversion)) + elif type == 'svn': + ud = bb.fetch2.FetchData(uri, d) + + svnFetcher = bb.fetch2.svn.Svn(d) + svnFetcher.urldata_init(ud, d) + try: + pupver = svnFetcher.latest_revision(ud, d, ud.names[0]) + except bb.fetch2.FetchError: + pstatus = "ErrSvnAccess" + + if pupver: + if pupver in pversion: + pstatus = "MATCH" + else: + pstatus = "UPDATE" + else: + pstatus = "ErrSvnAccess" + + if 'rev' in ud.parm: + pcurver = ud.parm['rev'] + + if pstatus != "ErrSvnAccess": + tag = pversion.rsplit("+svn")[0] + svn_prefix = re.search('(\+svn[r|\-]?)', pversion) + if tag and svn_prefix: + pupver = tag + svn_prefix.group() + pupver + + elif type == 'cvs': + pupver = "HEAD" + pstatus = "UPDATE" + elif type == 'file': + """local file is always up-to-date""" + pupver = pcurver + pstatus = "MATCH" + else: + pstatus = "ErrUnsupportedProto" + + if re.match("Err", pstatus): + pstatus += ":%s%s" % (host, path) + + """Read from manual distro tracking fields as alternative""" + pmver = d.getVar("RECIPE_UPSTREAM_VERSION", True) + if not pmver: + pmver = "N/A" + pmstatus = "ErrNoRecipeData" + else: + if pmver == pcurver: + pmstatus = "MATCH" + else: + pmstatus = "UPDATE" + + psrcuri = psrcuri.split()[0] + pdepends = "".join(pdepends.split("\t")) + pdesc = "".join(pdesc.split("\t")) + no_upgr_reason = d.getVar('RECIPE_NO_UPDATE_REASON', True) + lf = bb.utils.lockfile("%s.lock" % logfile) + f = open(logfile, "a") + f.write("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % \ + (pname,pversion,pupver,plicense,psection, phome,prelease, pdepends,pbugtracker,ppe,pdesc,pstatus,pmver,psrcuri,maintainer, no_upgr_reason)) + f.close() + bb.utils.unlockfile(lf) +} + +addtask checkpkgall after do_checkpkg +do_checkpkgall[recrdeptask] = "do_checkpkgall do_checkpkg" +do_checkpkgall[recideptask] = "do_${BB_DEFAULT_TASK}" +do_checkpkgall[nostamp] = "1" +do_checkpkgall() { + : +} + +addhandler distro_check_eventhandler +distro_check_eventhandler[eventmask] = "bb.event.BuildStarted" +python distro_check_eventhandler() { + """initialize log files.""" + import oe.distro_check as dc + result_file = dc.create_log_file(e.data, "distrocheck.csv") + return +} + +addtask distro_check +do_distro_check[nostamp] = "1" +python do_distro_check() { + """checks if the package is present in other public Linux distros""" + import oe.distro_check as dc + import shutil + if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('nativesdk',d): + return + + localdata = bb.data.createCopy(d) + bb.data.update_data(localdata) + tmpdir = d.getVar('TMPDIR', True) + distro_check_dir = os.path.join(tmpdir, "distro_check") + logpath = d.getVar('LOG_DIR', True) + bb.utils.mkdirhier(logpath) + result_file = os.path.join(logpath, "distrocheck.csv") + datetime = localdata.getVar('DATETIME', True) + dc.update_distro_data(distro_check_dir, datetime) + + # do the comparison + result = dc.compare_in_distro_packages_list(distro_check_dir, d) + + # save the results + dc.save_distro_check_result(result, datetime, result_file, d) +} + +addtask distro_checkall after do_distro_check +do_distro_checkall[recrdeptask] = "do_distro_checkall do_distro_check" +do_distro_checkall[recideptask] = "do_${BB_DEFAULT_TASK}" +do_distro_checkall[nostamp] = "1" +do_distro_checkall() { + : +} +# +#Check Missing License Text. +#Use this task to generate the missing license text data for pkg-report system, +#then we can search those recipes which license text isn't exsit in common-licenses directory +# +addhandler checklicense_eventhandler +checklicense_eventhandler[eventmask] = "bb.event.BuildStarted" +python checklicense_eventhandler() { + """initialize log files.""" + import oe.distro_check as dc + logfile = dc.create_log_file(e.data, "missinglicense.csv") + lf = bb.utils.lockfile("%s.lock" % logfile) + f = open(logfile, "a") + f.write("Package\tLicense\tMissingLicense\n") + f.close() + bb.utils.unlockfile(lf) + return +} + +addtask checklicense +do_checklicense[nostamp] = "1" +python do_checklicense() { + import shutil + logpath = d.getVar('LOG_DIR', True) + bb.utils.mkdirhier(logpath) + pn = d.getVar('PN', True) + logfile = os.path.join(logpath, "missinglicense.csv") + generic_directory = d.getVar('COMMON_LICENSE_DIR', True) + license_types = d.getVar('LICENSE', True) + for license_type in ((license_types.replace('+', '').replace('|', '&') + .replace('(', '').replace(')', '').replace(';', '') + .replace(',', '').replace(" ", "").split("&"))): + if not os.path.isfile(os.path.join(generic_directory, license_type)): + lf = bb.utils.lockfile("%s.lock" % logfile) + f = open(logfile, "a") + f.write("%s\t%s\t%s\n" % \ + (pn,license_types,license_type)) + f.close() + bb.utils.unlockfile(lf) + return +} + +addtask checklicenseall after do_checklicense +do_checklicenseall[recrdeptask] = "do_checklicenseall do_checklicense" +do_checklicenseall[recideptask] = "do_${BB_DEFAULT_TASK}" +do_checklicenseall[nostamp] = "1" +do_checklicenseall() { + : +} + + -- cgit v1.2.3-54-g00ecf