diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2011-02-04 10:26:21 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2011-02-07 09:06:36 +0000 |
commit | d08397ba4d1331993300eacbb2f78fcfef19c1cf (patch) | |
tree | 374d8aa46a69fbae08d2b3e8e7456a5c61005e4c /bitbake/lib/bb/fetch2/__init__.py | |
parent | f6eefb3ca3bb2a5ea0ec1364bdb0bc41ae58c815 (diff) | |
download | poky-d08397ba4d1331993300eacbb2f78fcfef19c1cf.tar.gz |
bitbake/fetch2: Rewrite and improve exception handling, reusing core functions for common operations where possible
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/fetch2/__init__.py')
-rw-r--r-- | bitbake/lib/bb/fetch2/__init__.py | 116 |
1 files changed, 78 insertions, 38 deletions
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index 54812eec43..10c1e73554 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py | |||
@@ -38,28 +38,62 @@ __version__ = "2" | |||
38 | logger = logging.getLogger("BitBake.Fetch") | 38 | logger = logging.getLogger("BitBake.Fetch") |
39 | 39 | ||
40 | class BBFetchException(Exception): | 40 | class BBFetchException(Exception): |
41 | """Class all fetch exceptions inherit from""" | 41 | """Class all fetch exceptions inherit from""" |
42 | def __init__(self, message): | ||
43 | self.msg = message | ||
44 | Exception.__init__(self, message) | ||
45 | |||
46 | def __str__(self): | ||
47 | return self.msg | ||
42 | 48 | ||
43 | class MalformedUrl(BBFetchException): | 49 | class MalformedUrl(BBFetchException): |
44 | """Exception raised when encountering an invalid url""" | 50 | """Exception raised when encountering an invalid url""" |
51 | def __init__(self, url): | ||
52 | self.msg = "The URL: '%s' is invalid and cannot be interpreted" % url | ||
53 | self.url = url | ||
54 | Exception.__init__(self, self.msg) | ||
45 | 55 | ||
46 | class FetchError(BBFetchException): | 56 | class FetchError(BBFetchException): |
47 | """Exception raised when a download fails""" | 57 | """General fetcher exception when something happens incorrectly""" |
58 | def __init__(self, message, url = None): | ||
59 | self.msg = "Fetcher failure for URL: '%s'. %s" % (url, message) | ||
60 | self.url = url | ||
61 | Exception.__init__(self, self.msg) | ||
48 | 62 | ||
49 | class NoMethodError(BBFetchException): | 63 | class NoMethodError(BBFetchException): |
50 | """Exception raised when there is no method to obtain a supplied url or set of urls""" | 64 | """Exception raised when there is no method to obtain a supplied url or set of urls""" |
65 | def __init__(self, url): | ||
66 | self.msg = "Could not find a fetcher which supports the URL: '%s'" % url | ||
67 | self.url = url | ||
68 | Exception.__init__(self, self.msg) | ||
51 | 69 | ||
52 | class MissingParameterError(BBFetchException): | 70 | class MissingParameterError(BBFetchException): |
53 | """Exception raised when a fetch method is missing a critical parameter in the url""" | 71 | """Exception raised when a fetch method is missing a critical parameter in the url""" |
72 | def __init__(self, missing, url): | ||
73 | self.msg = "URL: '%s' is missing the required parameter '%s'" % (url, missing) | ||
74 | self.url = url | ||
75 | self.missing = missing | ||
76 | Exception.__init__(self, self.msg) | ||
54 | 77 | ||
55 | class ParameterError(BBFetchException): | 78 | class ParameterError(BBFetchException): |
56 | """Exception raised when a url cannot be proccessed due to invalid parameters.""" | 79 | """Exception raised when a url cannot be proccessed due to invalid parameters.""" |
80 | def __init__(self, message, url): | ||
81 | self.msg = "URL: '%s' has invalid parameters. %s" % (url, message) | ||
82 | self.url = url | ||
83 | Exception.__init__(self, self.msg) | ||
57 | 84 | ||
58 | class MD5SumError(BBFetchException): | 85 | class MD5SumError(BBFetchException): |
59 | """Exception raised when a MD5SUM of a file does not match the expected one""" | 86 | """Exception raised when a MD5 checksum of a file does not match for a downloaded file""" |
60 | 87 | def __init__(self, path, wanted, got, url): | |
61 | class InvalidSRCREV(BBFetchException): | 88 | self.msg = "File: '%s' has md5 sum %s when %s was expected (from URL: '%s')" % (path, got, wanted, url) |
62 | """Exception raised when an invalid SRCREV is encountered""" | 89 | self.url = url |
90 | self.path = path | ||
91 | self.wanted = wanted | ||
92 | self.got = got | ||
93 | Exception.__init__(self, self.msg) | ||
94 | |||
95 | class SHA256SumError(MD5SumError): | ||
96 | """Exception raised when a SHA256 checksum of a file does not match for a downloaded file""" | ||
63 | 97 | ||
64 | def decodeurl(url): | 98 | def decodeurl(url): |
65 | """Decodes an URL into the tokens (scheme, network location, path, | 99 | """Decodes an URL into the tokens (scheme, network location, path, |
@@ -108,8 +142,10 @@ def encodeurl(decoded): | |||
108 | 142 | ||
109 | (type, host, path, user, pswd, p) = decoded | 143 | (type, host, path, user, pswd, p) = decoded |
110 | 144 | ||
111 | if not type or not path: | 145 | if not path: |
112 | raise MissingParameterError("Type or path url components missing when encoding %s" % decoded) | 146 | raise MissingParameterError('path', "encoded from the data %s" % str(decoded)) |
147 | if not type: | ||
148 | raise MissingParameterError('type', "encoded from the data %s" % str(decoded)) | ||
113 | url = '%s://' % type | 149 | url = '%s://' % type |
114 | if user and type != "file": | 150 | if user and type != "file": |
115 | url += "%s" % user | 151 | url += "%s" % user |
@@ -243,16 +279,14 @@ def verify_checksum(u, ud, d): | |||
243 | ud.localpath, ud.md5_name, md5data, | 279 | ud.localpath, ud.md5_name, md5data, |
244 | ud.sha256_name, sha256data) | 280 | ud.sha256_name, sha256data) |
245 | if bb.data.getVar("BB_STRICT_CHECKSUM", d, True) == "1": | 281 | if bb.data.getVar("BB_STRICT_CHECKSUM", d, True) == "1": |
246 | raise FetchError("No checksum specified for %s." % u) | 282 | raise FetchError("No checksum specified for %s." % u, u) |
247 | return | 283 | return |
248 | 284 | ||
249 | if (ud.md5_expected != md5data or ud.sha256_expected != sha256data): | 285 | if ud.md5_expected != md5data: |
250 | logger.error('The checksums for "%s" did not match.\n' | 286 | raise MD5SumError(ud.localpath, ud.md5_expected, md5data, u) |
251 | ' MD5: expected "%s", got "%s"\n' | 287 | |
252 | ' SHA256: expected "%s", got "%s"\n', | 288 | if ud.sha256_expected != sha256data: |
253 | ud.localpath, ud.md5_expected, md5data, | 289 | raise SHA256SumError(ud.localpath, ud.sha256_expected, sha256data, u) |
254 | ud.sha256_expected, sha256data) | ||
255 | raise FetchError("%s checksum mismatch." % u) | ||
256 | 290 | ||
257 | def subprocess_setup(): | 291 | def subprocess_setup(): |
258 | import signal | 292 | import signal |
@@ -319,7 +353,7 @@ def download(d, urls = None): | |||
319 | localpath = try_mirrors (d, u, mirrors) | 353 | localpath = try_mirrors (d, u, mirrors) |
320 | 354 | ||
321 | if not localpath or not os.path.exists(localpath): | 355 | if not localpath or not os.path.exists(localpath): |
322 | raise FetchError("Unable to fetch URL %s from any source." % u) | 356 | raise FetchError("Unable to fetch URL %s from any source." % u, u) |
323 | 357 | ||
324 | download_update(localpath, ud.localpath) | 358 | download_update(localpath, ud.localpath) |
325 | 359 | ||
@@ -365,7 +399,7 @@ def checkstatus(d, urls = None): | |||
365 | ret = try_mirrors (d, u, mirrors, True) | 399 | ret = try_mirrors (d, u, mirrors, True) |
366 | 400 | ||
367 | if not ret: | 401 | if not ret: |
368 | raise FetchError("URL %s doesn't work" % u) | 402 | raise FetchError("URL %s doesn't work" % u, u) |
369 | 403 | ||
370 | def localpaths(d): | 404 | def localpaths(d): |
371 | """ | 405 | """ |
@@ -403,8 +437,7 @@ def get_srcrev(d): | |||
403 | scms.append(u) | 437 | scms.append(u) |
404 | 438 | ||
405 | if len(scms) == 0: | 439 | if len(scms) == 0: |
406 | logger.error("SRCREV was used yet no valid SCM was found in SRC_URI") | 440 | raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI") |
407 | raise ParameterError | ||
408 | 441 | ||
409 | if len(scms) == 1 and len(urldata[scms[0]].names) == 1: | 442 | if len(scms) == 1 and len(urldata[scms[0]].names) == 1: |
410 | return urldata[scms[0]].method.sortable_revision(scms[0], urldata[scms[0]], d, urldata[scms[0]].names[0]) | 443 | return urldata[scms[0]].method.sortable_revision(scms[0], urldata[scms[0]], d, urldata[scms[0]].names[0]) |
@@ -414,8 +447,7 @@ def get_srcrev(d): | |||
414 | # | 447 | # |
415 | format = bb.data.getVar('SRCREV_FORMAT', d, 1) | 448 | format = bb.data.getVar('SRCREV_FORMAT', d, 1) |
416 | if not format: | 449 | if not format: |
417 | logger.error("The SRCREV_FORMAT variable must be set when multiple SCMs are used.") | 450 | raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.") |
418 | raise ParameterError | ||
419 | 451 | ||
420 | for scm in scms: | 452 | for scm in scms: |
421 | ud = urldata[scm] | 453 | ud = urldata[scm] |
@@ -435,11 +467,12 @@ def localpath(url, d): | |||
435 | return ud[url].localpath | 467 | return ud[url].localpath |
436 | return url | 468 | return url |
437 | 469 | ||
438 | def runfetchcmd(cmd, d, quiet = False): | 470 | def runfetchcmd(cmd, d, quiet = False, cleanup = []): |
439 | """ | 471 | """ |
440 | Run cmd returning the command output | 472 | Run cmd returning the command output |
441 | Raise an error if interrupted or cmd fails | 473 | Raise an error if interrupted or cmd fails |
442 | Optionally echo command output to stdout | 474 | Optionally echo command output to stdout |
475 | Optionally remove the files/directories listed in cleanup upon failure | ||
443 | """ | 476 | """ |
444 | 477 | ||
445 | # Need to export PATH as binary could be in metadata paths | 478 | # Need to export PATH as binary could be in metadata paths |
@@ -474,10 +507,17 @@ def runfetchcmd(cmd, d, quiet = False): | |||
474 | signal = status >> 8 | 507 | signal = status >> 8 |
475 | exitstatus = status & 0xff | 508 | exitstatus = status & 0xff |
476 | 509 | ||
477 | if signal: | 510 | if (signal or status != 0): |
478 | raise FetchError("Fetch command %s failed with signal %s, output:\n%s" % (cmd, signal, output)) | 511 | for f in cleanup: |
479 | elif status != 0: | 512 | try: |
480 | raise FetchError("Fetch command %s failed with exit code %s, output:\n%s" % (cmd, status, output)) | 513 | bb.utils.remove(f, True) |
514 | except OSError: | ||
515 | pass | ||
516 | |||
517 | if signal: | ||
518 | raise FetchError("Fetch command %s failed with signal %s, output:\n%s" % (cmd, signal, output)) | ||
519 | elif status != 0: | ||
520 | raise FetchError("Fetch command %s failed with exit code %s, output:\n%s" % (cmd, status, output)) | ||
481 | 521 | ||
482 | return output | 522 | return output |
483 | 523 | ||
@@ -486,10 +526,9 @@ def check_network_access(d, info = ""): | |||
486 | log remote network access, and error if BB_NO_NETWORK is set | 526 | log remote network access, and error if BB_NO_NETWORK is set |
487 | """ | 527 | """ |
488 | if bb.data.getVar("BB_NO_NETWORK", d, True) == "1": | 528 | if bb.data.getVar("BB_NO_NETWORK", d, True) == "1": |
489 | bb.error("BB_NO_NETWORK is set, but the fetcher code attempted network access with the command %s" % info) | 529 | raise FetchError("BB_NO_NETWORK is set, but the fetcher code attempted network access with the command %s" % info) |
490 | raise FetchError("BB_NO_NETWORK violation") | ||
491 | else: | 530 | else: |
492 | bb.note("Fetcher accessed the network with the command %s" % info) | 531 | logger.debug(1, "Fetcher accessed the network with the command %s" % info) |
493 | 532 | ||
494 | def try_mirrors(d, uri, mirrors, check = False, force = False): | 533 | def try_mirrors(d, uri, mirrors, check = False, force = False): |
495 | """ | 534 | """ |
@@ -573,7 +612,7 @@ class FetchData(object): | |||
573 | break | 612 | break |
574 | 613 | ||
575 | if not self.method: | 614 | if not self.method: |
576 | raise NoMethodError("Missing implementation for url %s" % url) | 615 | raise NoMethodError(url) |
577 | 616 | ||
578 | if self.method.supports_srcrev(): | 617 | if self.method.supports_srcrev(): |
579 | self.revisions = {} | 618 | self.revisions = {} |
@@ -658,7 +697,7 @@ class Fetch(object): | |||
658 | Fetch urls | 697 | Fetch urls |
659 | Assumes localpath was called first | 698 | Assumes localpath was called first |
660 | """ | 699 | """ |
661 | raise NoMethodError("Missing implementation for url") | 700 | raise NoMethodError(url) |
662 | 701 | ||
663 | def unpack(self, urldata, rootdir, data): | 702 | def unpack(self, urldata, rootdir, data): |
664 | import subprocess | 703 | import subprocess |
@@ -795,7 +834,7 @@ class Fetch(object): | |||
795 | if not rev: | 834 | if not rev: |
796 | rev = data.getVar("SRCREV", d, 1) | 835 | rev = data.getVar("SRCREV", d, 1) |
797 | if rev == "INVALID": | 836 | if rev == "INVALID": |
798 | raise InvalidSRCREV("Please set SRCREV to a valid value") | 837 | raise FetchError("Please set SRCREV to a valid value", ud.url) |
799 | if rev == "AUTOINC": | 838 | if rev == "AUTOINC": |
800 | rev = ud.method.latest_revision(ud.url, ud, d, name) | 839 | rev = ud.method.latest_revision(ud.url, ud, d, name) |
801 | 840 | ||
@@ -828,14 +867,15 @@ class Fetch(object): | |||
828 | if not wanted_sum: | 867 | if not wanted_sum: |
829 | return True | 868 | return True |
830 | 869 | ||
831 | return wanted_sum == got_sum | 870 | if wanted_sum != got_sum: |
871 | raise MD5SumError(ud.localpath, wanted_sum, got_sum, ud.url) | ||
872 | |||
832 | verify_md5sum = staticmethod(verify_md5sum) | 873 | verify_md5sum = staticmethod(verify_md5sum) |
833 | 874 | ||
834 | def write_md5sum(url, ud, d): | 875 | def write_md5sum(url, ud, d): |
835 | md5data = bb.utils.md5_file(ud.localpath) | 876 | md5data = bb.utils.md5_file(ud.localpath) |
836 | # verify the md5sum | 877 | |
837 | if not Fetch.verify_md5sum(ud, md5data): | 878 | Fetch.verify_md5sum(ud, md5data) |
838 | raise MD5SumError(url) | ||
839 | 879 | ||
840 | md5out = file(ud.md5, 'w') | 880 | md5out = file(ud.md5, 'w') |
841 | md5out.write(md5data) | 881 | md5out.write(md5data) |
@@ -847,7 +887,7 @@ class Fetch(object): | |||
847 | Look in the cache for the latest revision, if not present ask the SCM. | 887 | Look in the cache for the latest revision, if not present ask the SCM. |
848 | """ | 888 | """ |
849 | if not hasattr(self, "_latest_revision"): | 889 | if not hasattr(self, "_latest_revision"): |
850 | raise ParameterError | 890 | raise ParameterError("The fetcher for this URL does not support _latest_revision", url) |
851 | 891 | ||
852 | pd = persist_data.persist(d) | 892 | pd = persist_data.persist(d) |
853 | revs = pd['BB_URI_HEADREVS'] | 893 | revs = pd['BB_URI_HEADREVS'] |