From 9f22898e0f9155a75c6b6f44869b129e505910e8 Mon Sep 17 00:00:00 2001 From: Ross Burton Date: Wed, 20 Jan 2016 13:10:25 +0000 Subject: bitbake: fetch2/wget: fallback to GET if HEAD is rejected in checkstatus() The core change here is to fall back to GET requests if HEAD is rejected in the checkstatus() method, as you can't do a HEAD on Amazon S3 (used by Github archives). This meant removing the monkey patch that the default method was GET and adding a fixed redirect handler that doesn't reset to GET. Also, change the way the opener is constructed from an if/elif cluster to a conditionally constructed list. (Bitbake rev: 6ec70d5d2e330b41b932b0a655b838a5f37df01e) Signed-off-by: Ross Burton Signed-off-by: Richard Purdie --- bitbake/lib/bb/fetch2/wget.py | 72 +++++++++++++++++++++++++++++-------------- bitbake/lib/bb/tests/fetch.py | 2 ++ 2 files changed, 51 insertions(+), 23 deletions(-) (limited to 'bitbake') diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py index c8c6d5ce83..5a31730a4a 100644 --- a/bitbake/lib/bb/fetch2/wget.py +++ b/bitbake/lib/bb/fetch2/wget.py @@ -235,38 +235,64 @@ class Wget(FetchMethod): return exported - def head_method(self): - return "HEAD" - + class HTTPMethodFallback(urllib2.BaseHandler): + """ + Fallback to GET if HEAD is not allowed (405 HTTP error) + """ + def http_error_405(self, req, fp, code, msg, headers): + fp.read() + fp.close() + + newheaders = dict((k,v) for k,v in req.headers.items() + if k.lower() not in ("content-length", "content-type")) + return self.parent.open(urllib2.Request(req.get_full_url(), + headers=newheaders, + origin_req_host=req.get_origin_req_host(), + unverifiable=True)) + + """ + Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403 + Forbidden when they actually mean 405 Method Not Allowed. + """ + http_error_403 = http_error_405 + + """ + Some servers (e.g. FusionForge) returns 406 Not Acceptable when they + actually mean 405 Method Not Allowed. + """ + http_error_406 = http_error_405 + + class FixedHTTPRedirectHandler(urllib2.HTTPRedirectHandler): + """ + urllib2.HTTPRedirectHandler resets the method to GET on redirect, + when we want to follow redirects using the original method. + """ + def redirect_request(self, req, fp, code, msg, headers, newurl): + newreq = urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) + newreq.get_method = lambda: req.get_method() + return newreq exported_proxies = export_proxies(d) + handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] + if export_proxies: + handlers.append(urllib2.ProxyHandler()) + handlers.append(CacheHTTPHandler()) # XXX: Since Python 2.7.9 ssl cert validation is enabled by default # see PEP-0476, this causes verification errors on some https servers # so disable by default. import ssl - ssl_context = None if hasattr(ssl, '_create_unverified_context'): - ssl_context = ssl._create_unverified_context() - - if exported_proxies == True and ssl_context is not None: - opener = urllib2.build_opener(urllib2.ProxyHandler, CacheHTTPHandler, - urllib2.HTTPSHandler(context=ssl_context)) - elif exported_proxies == False and ssl_context is not None: - opener = urllib2.build_opener(CacheHTTPHandler, - urllib2.HTTPSHandler(context=ssl_context)) - elif exported_proxies == True and ssl_context is None: - opener = urllib2.build_opener(urllib2.ProxyHandler, CacheHTTPHandler) - else: - opener = urllib2.build_opener(CacheHTTPHandler) - - urllib2.Request.get_method = head_method - urllib2.install_opener(opener) - - uri = ud.url.split(";")[0] + handlers.append(urllib2.HTTPSHandler(context=ssl._create_unverified_context())) + opener = urllib2.build_opener(*handlers) try: - urllib2.urlopen(uri) - except: + uri = ud.url.split(";")[0] + r = urllib2.Request(uri) + r.get_method = lambda: "HEAD" + opener.open(r) + except urllib2.URLError as e: + # debug for now to avoid spamming the logs in e.g. remote sstate searches + logger.debug(2, "checkstatus() urlopen failed: %s" % e) return False return True diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py index b175a43280..81b22d025f 100644 --- a/bitbake/lib/bb/tests/fetch.py +++ b/bitbake/lib/bb/tests/fetch.py @@ -738,6 +738,8 @@ class FetchCheckStatusTest(FetcherTest): "ftp://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz", "ftp://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz", "ftp://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz", + # GitHub releases are hosted on Amazon S3, which doesn't support HEAD + "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz" ] if os.environ.get("BB_SKIP_NETTESTS") == "yes": -- cgit v1.2.3-54-g00ecf