From d08397ba4d1331993300eacbb2f78fcfef19c1cf Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Fri, 4 Feb 2011 10:26:21 +0000 Subject: bitbake/fetch2: Rewrite and improve exception handling, reusing core functions for common operations where possible Signed-off-by: Richard Purdie --- bitbake/lib/bb/fetch2/__init__.py | 116 +++++++++++++++++++++++++------------- bitbake/lib/bb/fetch2/bzr.py | 12 +--- bitbake/lib/bb/fetch2/cvs.py | 30 ++++------ bitbake/lib/bb/fetch2/git.py | 4 +- bitbake/lib/bb/fetch2/hg.py | 14 +---- bitbake/lib/bb/fetch2/osc.py | 14 +---- bitbake/lib/bb/fetch2/perforce.py | 20 ++----- bitbake/lib/bb/fetch2/ssh.py | 6 +- bitbake/lib/bb/fetch2/svk.py | 23 ++------ bitbake/lib/bb/fetch2/svn.py | 14 +---- bitbake/lib/bb/fetch2/wget.py | 13 ++--- 11 files changed, 121 insertions(+), 145 deletions(-) diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index 54812eec43..10c1e73554 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py @@ -38,28 +38,62 @@ __version__ = "2" logger = logging.getLogger("BitBake.Fetch") class BBFetchException(Exception): - """Class all fetch exceptions inherit from""" + """Class all fetch exceptions inherit from""" + def __init__(self, message): + self.msg = message + Exception.__init__(self, message) + + def __str__(self): + return self.msg class MalformedUrl(BBFetchException): """Exception raised when encountering an invalid url""" + def __init__(self, url): + self.msg = "The URL: '%s' is invalid and cannot be interpreted" % url + self.url = url + Exception.__init__(self, self.msg) class FetchError(BBFetchException): - """Exception raised when a download fails""" + """General fetcher exception when something happens incorrectly""" + def __init__(self, message, url = None): + self.msg = "Fetcher failure for URL: '%s'. %s" % (url, message) + self.url = url + Exception.__init__(self, self.msg) class NoMethodError(BBFetchException): """Exception raised when there is no method to obtain a supplied url or set of urls""" + def __init__(self, url): + self.msg = "Could not find a fetcher which supports the URL: '%s'" % url + self.url = url + Exception.__init__(self, self.msg) class MissingParameterError(BBFetchException): """Exception raised when a fetch method is missing a critical parameter in the url""" + def __init__(self, missing, url): + self.msg = "URL: '%s' is missing the required parameter '%s'" % (url, missing) + self.url = url + self.missing = missing + Exception.__init__(self, self.msg) class ParameterError(BBFetchException): """Exception raised when a url cannot be proccessed due to invalid parameters.""" + def __init__(self, message, url): + self.msg = "URL: '%s' has invalid parameters. %s" % (url, message) + self.url = url + Exception.__init__(self, self.msg) class MD5SumError(BBFetchException): - """Exception raised when a MD5SUM of a file does not match the expected one""" - -class InvalidSRCREV(BBFetchException): - """Exception raised when an invalid SRCREV is encountered""" + """Exception raised when a MD5 checksum of a file does not match for a downloaded file""" + def __init__(self, path, wanted, got, url): + self.msg = "File: '%s' has md5 sum %s when %s was expected (from URL: '%s')" % (path, got, wanted, url) + self.url = url + self.path = path + self.wanted = wanted + self.got = got + Exception.__init__(self, self.msg) + +class SHA256SumError(MD5SumError): + """Exception raised when a SHA256 checksum of a file does not match for a downloaded file""" def decodeurl(url): """Decodes an URL into the tokens (scheme, network location, path, @@ -108,8 +142,10 @@ def encodeurl(decoded): (type, host, path, user, pswd, p) = decoded - if not type or not path: - raise MissingParameterError("Type or path url components missing when encoding %s" % decoded) + if not path: + raise MissingParameterError('path', "encoded from the data %s" % str(decoded)) + if not type: + raise MissingParameterError('type', "encoded from the data %s" % str(decoded)) url = '%s://' % type if user and type != "file": url += "%s" % user @@ -243,16 +279,14 @@ def verify_checksum(u, ud, d): ud.localpath, ud.md5_name, md5data, ud.sha256_name, sha256data) if bb.data.getVar("BB_STRICT_CHECKSUM", d, True) == "1": - raise FetchError("No checksum specified for %s." % u) + raise FetchError("No checksum specified for %s." % u, u) return - if (ud.md5_expected != md5data or ud.sha256_expected != sha256data): - logger.error('The checksums for "%s" did not match.\n' - ' MD5: expected "%s", got "%s"\n' - ' SHA256: expected "%s", got "%s"\n', - ud.localpath, ud.md5_expected, md5data, - ud.sha256_expected, sha256data) - raise FetchError("%s checksum mismatch." % u) + if ud.md5_expected != md5data: + raise MD5SumError(ud.localpath, ud.md5_expected, md5data, u) + + if ud.sha256_expected != sha256data: + raise SHA256SumError(ud.localpath, ud.sha256_expected, sha256data, u) def subprocess_setup(): import signal @@ -319,7 +353,7 @@ def download(d, urls = None): localpath = try_mirrors (d, u, mirrors) if not localpath or not os.path.exists(localpath): - raise FetchError("Unable to fetch URL %s from any source." % u) + raise FetchError("Unable to fetch URL %s from any source." % u, u) download_update(localpath, ud.localpath) @@ -365,7 +399,7 @@ def checkstatus(d, urls = None): ret = try_mirrors (d, u, mirrors, True) if not ret: - raise FetchError("URL %s doesn't work" % u) + raise FetchError("URL %s doesn't work" % u, u) def localpaths(d): """ @@ -403,8 +437,7 @@ def get_srcrev(d): scms.append(u) if len(scms) == 0: - logger.error("SRCREV was used yet no valid SCM was found in SRC_URI") - raise ParameterError + raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI") if len(scms) == 1 and len(urldata[scms[0]].names) == 1: return urldata[scms[0]].method.sortable_revision(scms[0], urldata[scms[0]], d, urldata[scms[0]].names[0]) @@ -414,8 +447,7 @@ def get_srcrev(d): # format = bb.data.getVar('SRCREV_FORMAT', d, 1) if not format: - logger.error("The SRCREV_FORMAT variable must be set when multiple SCMs are used.") - raise ParameterError + raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.") for scm in scms: ud = urldata[scm] @@ -435,11 +467,12 @@ def localpath(url, d): return ud[url].localpath return url -def runfetchcmd(cmd, d, quiet = False): +def runfetchcmd(cmd, d, quiet = False, cleanup = []): """ Run cmd returning the command output Raise an error if interrupted or cmd fails Optionally echo command output to stdout + Optionally remove the files/directories listed in cleanup upon failure """ # Need to export PATH as binary could be in metadata paths @@ -474,10 +507,17 @@ def runfetchcmd(cmd, d, quiet = False): signal = status >> 8 exitstatus = status & 0xff - if signal: - raise FetchError("Fetch command %s failed with signal %s, output:\n%s" % (cmd, signal, output)) - elif status != 0: - raise FetchError("Fetch command %s failed with exit code %s, output:\n%s" % (cmd, status, output)) + if (signal or status != 0): + for f in cleanup: + try: + bb.utils.remove(f, True) + except OSError: + pass + + if signal: + raise FetchError("Fetch command %s failed with signal %s, output:\n%s" % (cmd, signal, output)) + elif status != 0: + raise FetchError("Fetch command %s failed with exit code %s, output:\n%s" % (cmd, status, output)) return output @@ -486,10 +526,9 @@ def check_network_access(d, info = ""): log remote network access, and error if BB_NO_NETWORK is set """ if bb.data.getVar("BB_NO_NETWORK", d, True) == "1": - bb.error("BB_NO_NETWORK is set, but the fetcher code attempted network access with the command %s" % info) - raise FetchError("BB_NO_NETWORK violation") + raise FetchError("BB_NO_NETWORK is set, but the fetcher code attempted network access with the command %s" % info) else: - bb.note("Fetcher accessed the network with the command %s" % info) + logger.debug(1, "Fetcher accessed the network with the command %s" % info) def try_mirrors(d, uri, mirrors, check = False, force = False): """ @@ -573,7 +612,7 @@ class FetchData(object): break if not self.method: - raise NoMethodError("Missing implementation for url %s" % url) + raise NoMethodError(url) if self.method.supports_srcrev(): self.revisions = {} @@ -658,7 +697,7 @@ class Fetch(object): Fetch urls Assumes localpath was called first """ - raise NoMethodError("Missing implementation for url") + raise NoMethodError(url) def unpack(self, urldata, rootdir, data): import subprocess @@ -795,7 +834,7 @@ class Fetch(object): if not rev: rev = data.getVar("SRCREV", d, 1) if rev == "INVALID": - raise InvalidSRCREV("Please set SRCREV to a valid value") + raise FetchError("Please set SRCREV to a valid value", ud.url) if rev == "AUTOINC": rev = ud.method.latest_revision(ud.url, ud, d, name) @@ -828,14 +867,15 @@ class Fetch(object): if not wanted_sum: return True - return wanted_sum == got_sum + if wanted_sum != got_sum: + raise MD5SumError(ud.localpath, wanted_sum, got_sum, ud.url) + verify_md5sum = staticmethod(verify_md5sum) def write_md5sum(url, ud, d): md5data = bb.utils.md5_file(ud.localpath) - # verify the md5sum - if not Fetch.verify_md5sum(ud, md5data): - raise MD5SumError(url) + + Fetch.verify_md5sum(ud, md5data) md5out = file(ud.md5, 'w') md5out.write(md5data) @@ -847,7 +887,7 @@ class Fetch(object): Look in the cache for the latest revision, if not present ask the SCM. """ if not hasattr(self, "_latest_revision"): - raise ParameterError + raise ParameterError("The fetcher for this URL does not support _latest_revision", url) pd = persist_data.persist(d) revs = pd['BB_URI_HEADREVS'] diff --git a/bitbake/lib/bb/fetch2/bzr.py b/bitbake/lib/bb/fetch2/bzr.py index 1e25207b19..22168a8351 100644 --- a/bitbake/lib/bb/fetch2/bzr.py +++ b/bitbake/lib/bb/fetch2/bzr.py @@ -72,7 +72,7 @@ class Bzr(Fetch): elif command is "update": bzrcmd = "%s pull %s --overwrite" % (basecmd, " ".join(options)) else: - raise FetchError("Invalid bzr command %s" % command) + raise FetchError("Invalid bzr command %s" % command, ud.url) return bzrcmd @@ -104,15 +104,7 @@ class Bzr(Fetch): tar_flags = "--exclude '.bzr' --exclude '.bzrtags'" # tar them up to a defined filename - try: - runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d) - except: - t, v, tb = sys.exc_info() - try: - os.unlink(ud.localpath) - except OSError: - pass - raise t, v, tb + runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d, cleanup = [ud.localpath]) def supports_srcrev(self): return True diff --git a/bitbake/lib/bb/fetch2/cvs.py b/bitbake/lib/bb/fetch2/cvs.py index 907812d717..d792328c25 100644 --- a/bitbake/lib/bb/fetch2/cvs.py +++ b/bitbake/lib/bb/fetch2/cvs.py @@ -44,7 +44,7 @@ class Cvs(Fetch): def urldata_init(self, ud, d): if not "module" in ud.parm: - raise MissingParameterError("cvs method needs a 'module' parameter") + raise MissingParameterError("module", ud.url) ud.module = ud.parm["module"] ud.tag = ud.parm.get('tag', "") @@ -132,7 +132,7 @@ class Cvs(Fetch): bb.fetch2.check_network_access(d, cvsupdatecmd) # update sources there os.chdir(moddir) - myret = os.system(cvsupdatecmd) + cmd = cvsupdatecmd else: logger.info("Fetch " + loc) # check out sources there @@ -140,14 +140,12 @@ class Cvs(Fetch): os.chdir(pkgdir) logger.debug(1, "Running %s", cvscmd) bb.fetch2.check_network_access(d, cvscmd) - myret = os.system(cvscmd) + cmd = cvscmd - if myret != 0 or not os.access(moddir, os.R_OK): - try: - os.rmdir(moddir) - except OSError: - pass - raise FetchError(ud.module) + runfetchcmd(cmd, d, cleanup = [moddir]) + + if not os.access(moddir, os.R_OK): + raise FetchError("Directory %s was not readable despite sucessful fetch?!" % moddir, ud.url) scmdata = ud.parm.get("scmdata", "") if scmdata == "keep": @@ -158,15 +156,11 @@ class Cvs(Fetch): # tar them up to a defined filename if 'fullpath' in ud.parm: os.chdir(pkgdir) - myret = os.system("tar %s -czf %s %s" % (tar_flags, ud.localpath, localdir)) + cmd = "tar %s -czf %s %s" % (tar_flags, ud.localpath, localdir) else: os.chdir(moddir) os.chdir('..') - myret = os.system("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(moddir))) - - if myret != 0: - try: - os.unlink(ud.localpath) - except OSError: - pass - raise FetchError(ud.module) + cmd = "tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(moddir)) + + runfetchcmd(cmd, d, cleanup = [ud.localpath]) + diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index 392491ed87..1ae1d40302 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py @@ -59,7 +59,7 @@ class Git(Fetch): branches = ud.parm.get("branch", "master").split(',') if len(branches) != len(ud.names): - raise bb.fetch2.ParameterError("SRC_URI (%) name and branch number mismatch" % ud.url) + raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) ud.branches = {} for name in ud.names: branch = branches[ud.names.index(name)] @@ -194,7 +194,7 @@ class Git(Fetch): cmd = "%s ls-remote %s://%s%s%s %s" % (basecmd, ud.proto, username, ud.host, ud.path, ud.branches[name]) output = runfetchcmd(cmd, d, True) if not output: - raise bb.fetch2.FetchError("Fetch command %s gave empty output\n" % (cmd)) + raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, url) return output.split()[0] def _build_revision(self, url, ud, d, name): diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py index 6929d971f5..ffede8cf5a 100644 --- a/bitbake/lib/bb/fetch2/hg.py +++ b/bitbake/lib/bb/fetch2/hg.py @@ -48,7 +48,7 @@ class Hg(Fetch): init hg specific variable within url data """ if not "module" in ud.parm: - raise MissingParameterError("hg method needs a 'module' parameter") + raise MissingParameterError('module', ud.url) ud.module = ud.parm["module"] @@ -105,7 +105,7 @@ class Hg(Fetch): elif command is "update": cmd = "%s update -C %s" % (basecmd, " ".join(options)) else: - raise FetchError("Invalid hg command %s" % command) + raise FetchError("Invalid hg command %s" % command, ud.url) return cmd @@ -147,15 +147,7 @@ class Hg(Fetch): tar_flags = "--exclude '.hg' --exclude '.hgrags'" os.chdir(ud.pkgdir) - try: - runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d) - except: - t, v, tb = sys.exc_info() - try: - os.unlink(ud.localpath) - except OSError: - pass - raise t, v, tb + runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d, cleanup = [ud.localpath]) def supports_srcrev(self): return True diff --git a/bitbake/lib/bb/fetch2/osc.py b/bitbake/lib/bb/fetch2/osc.py index 14bde82fd7..9b97cf1cc7 100644 --- a/bitbake/lib/bb/fetch2/osc.py +++ b/bitbake/lib/bb/fetch2/osc.py @@ -28,7 +28,7 @@ class Osc(Fetch): def urldata_init(self, ud, d): if not "module" in ud.parm: - raise MissingParameterError("osc method needs a 'module' parameter.") + raise MissingParameterError('module', ud.url) ud.module = ud.parm["module"] @@ -73,7 +73,7 @@ class Osc(Fetch): elif command is "update": osccmd = "%s %s up %s" % (basecmd, config, " ".join(options)) else: - raise FetchError("Invalid osc command %s" % command) + raise FetchError("Invalid osc command %s" % command, ud.url) return osccmd @@ -104,15 +104,7 @@ class Osc(Fetch): os.chdir(os.path.join(ud.pkgdir + ud.path)) # tar them up to a defined filename - try: - runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d) - except: - t, v, tb = sys.exc_info() - try: - os.unlink(ud.localpath) - except OSError: - pass - raise t, v, tb + runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d, cleanup = [ud.localpath]) def supports_srcrev(self): return False diff --git a/bitbake/lib/bb/fetch2/perforce.py b/bitbake/lib/bb/fetch2/perforce.py index 5c128b3fde..e98440f59f 100644 --- a/bitbake/lib/bb/fetch2/perforce.py +++ b/bitbake/lib/bb/fetch2/perforce.py @@ -156,8 +156,7 @@ class Perforce(Fetch): tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false") tmpfile = tmppipe.readline().strip() if not tmpfile: - logger.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.") - raise FetchError(module) + raise FetchError("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.", loc) if "label" in parm: depot = "%s@%s" % (depot, parm["label"]) @@ -171,8 +170,7 @@ class Perforce(Fetch): p4file = os.popen("%s%s files %s" % (p4cmd, p4opt, depot)) if not p4file: - logger.error("Fetch: unable to get the P4 files from %s", depot) - raise FetchError(module) + raise FetchError("Fetch: unable to get the P4 files from %s" % depot, loc) count = 0 @@ -189,15 +187,9 @@ class Perforce(Fetch): count = count + 1 if count == 0: - logger.error("Fetch: No files gathered from the P4 fetch") - raise FetchError(module) - - myret = os.system("tar -czf %s %s" % (ud.localpath, module)) - if myret != 0: - try: - os.unlink(ud.localpath) - except OSError: - pass - raise FetchError(module) + logger.error() + raise FetchError("Fetch: No files gathered from the P4 fetch", loc) + + runfetchcmd("tar -czf %s %s" % (ud.localpath, module), d, cleanup = [ud.localpath]) # cleanup bb.utils.prunedir(tmpfile) diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py index 246439ef19..29cc272193 100644 --- a/bitbake/lib/bb/fetch2/ssh.py +++ b/bitbake/lib/bb/fetch2/ssh.py @@ -114,7 +114,5 @@ class SSH(Fetch): bb.fetch2.check_network_access(d, cmd) - (exitstatus, output) = commands.getstatusoutput(cmd) - if exitstatus != 0: - print(output) - raise FetchError('Unable to fetch %s' % url) + runfetchcmd(cmd, d) + diff --git a/bitbake/lib/bb/fetch2/svk.py b/bitbake/lib/bb/fetch2/svk.py index 95206f52fb..8220bf3dc2 100644 --- a/bitbake/lib/bb/fetch2/svk.py +++ b/bitbake/lib/bb/fetch2/svk.py @@ -45,7 +45,7 @@ class Svk(Fetch): def urldata_init(self, ud, d): if not "module" in ud.parm: - raise MissingParameterError("svk method needs a 'module' parameter") + raise MissingParameterError('module', ud.url) else: ud.module = ud.parm["module"] @@ -75,29 +75,18 @@ class Svk(Fetch): tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false") tmpfile = tmppipe.readline().strip() if not tmpfile: - logger.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.") - raise FetchError(ud.module) + logger.error() + raise FetchError("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.", loc) # check out sources there os.chdir(tmpfile) logger.info("Fetch " + loc) logger.debug(1, "Running %s", svkcmd) - myret = os.system(svkcmd) - if myret != 0: - try: - os.rmdir(tmpfile) - except OSError: - pass - raise FetchError(ud.module) + runfetchcmd(svkcmd, d, cleanup = [tmpfile]) os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module))) # tar them up to a defined filename - myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module))) - if myret != 0: - try: - os.unlink(ud.localpath) - except OSError: - pass - raise FetchError(ud.module) + runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module)), d, cleanup = [ud.localpath]) + # cleanup bb.utils.prunedir(tmpfile) diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py index 5ff064ebf3..267c9bb7e6 100644 --- a/bitbake/lib/bb/fetch2/svn.py +++ b/bitbake/lib/bb/fetch2/svn.py @@ -47,7 +47,7 @@ class Svn(Fetch): init svn specific variable within url data """ if not "module" in ud.parm: - raise MissingParameterError("svn method needs a 'module' parameter") + raise MissingParameterError('module', ud.url) ud.module = ud.parm["module"] @@ -118,7 +118,7 @@ class Svn(Fetch): elif command is "update": svncmd = "%s update %s" % (basecmd, " ".join(options)) else: - raise FetchError("Invalid svn command %s" % command) + raise FetchError("Invalid svn command %s" % command, ud.url) if svn_rsh: svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd) @@ -156,15 +156,7 @@ class Svn(Fetch): os.chdir(ud.pkgdir) # tar them up to a defined filename - try: - runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d) - except: - t, v, tb = sys.exc_info() - try: - os.unlink(ud.localpath) - except OSError: - pass - raise t, v, tb + runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d, cleanup = [ud.localpath]) def supports_srcrev(self): return True diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py index 8d623721a9..e33265e1cc 100644 --- a/bitbake/lib/bb/fetch2/wget.py +++ b/bitbake/lib/bb/fetch2/wget.py @@ -72,20 +72,15 @@ class Wget(Fetch): # Sanity check since wget can pretend it succeed when it didn't # Also, this used to happen if sourceforge sent us to the mirror page if not os.path.exists(ud.localpath) and not checkonly: - logger.debug(2, "The fetch command for %s returned success but %s doesn't exist?...", uri, ud.localpath) - return False - - return True + raise FetchError("The fetch command returned success but %s doesn't exist?!" % (uri, ud.localpath), uri) localdata = data.createCopy(d) data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata) data.update_data(localdata) - if fetch_uri(uri, ud, localdata): - return True - - raise FetchError(uri) - + fetch_uri(uri, ud, localdata) + + return True def checkstatus(self, uri, ud, d): return self.download(uri, ud, d, True) -- cgit v1.2.3-54-g00ecf