diff options
-rw-r--r-- | bitbake/lib/bb/fetch2/wget.py | 72 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/fetch.py | 2 |
2 files changed, 51 insertions, 23 deletions
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py index bd2a8972a7..0ff5b2ba33 100644 --- a/bitbake/lib/bb/fetch2/wget.py +++ b/bitbake/lib/bb/fetch2/wget.py | |||
@@ -234,38 +234,64 @@ class Wget(FetchMethod): | |||
234 | 234 | ||
235 | return exported | 235 | return exported |
236 | 236 | ||
237 | def head_method(self): | 237 | class HTTPMethodFallback(urllib2.BaseHandler): |
238 | return "HEAD" | 238 | """ |
239 | 239 | Fallback to GET if HEAD is not allowed (405 HTTP error) | |
240 | """ | ||
241 | def http_error_405(self, req, fp, code, msg, headers): | ||
242 | fp.read() | ||
243 | fp.close() | ||
244 | |||
245 | newheaders = dict((k,v) for k,v in req.headers.items() | ||
246 | if k.lower() not in ("content-length", "content-type")) | ||
247 | return self.parent.open(urllib2.Request(req.get_full_url(), | ||
248 | headers=newheaders, | ||
249 | origin_req_host=req.get_origin_req_host(), | ||
250 | unverifiable=True)) | ||
251 | |||
252 | """ | ||
253 | Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403 | ||
254 | Forbidden when they actually mean 405 Method Not Allowed. | ||
255 | """ | ||
256 | http_error_403 = http_error_405 | ||
257 | |||
258 | """ | ||
259 | Some servers (e.g. FusionForge) returns 406 Not Acceptable when they | ||
260 | actually mean 405 Method Not Allowed. | ||
261 | """ | ||
262 | http_error_406 = http_error_405 | ||
263 | |||
264 | class FixedHTTPRedirectHandler(urllib2.HTTPRedirectHandler): | ||
265 | """ | ||
266 | urllib2.HTTPRedirectHandler resets the method to GET on redirect, | ||
267 | when we want to follow redirects using the original method. | ||
268 | """ | ||
269 | def redirect_request(self, req, fp, code, msg, headers, newurl): | ||
270 | newreq = urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) | ||
271 | newreq.get_method = lambda: req.get_method() | ||
272 | return newreq | ||
240 | exported_proxies = export_proxies(d) | 273 | exported_proxies = export_proxies(d) |
241 | 274 | ||
275 | handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] | ||
276 | if export_proxies: | ||
277 | handlers.append(urllib2.ProxyHandler()) | ||
278 | handlers.append(CacheHTTPHandler()) | ||
242 | # XXX: Since Python 2.7.9 ssl cert validation is enabled by default | 279 | # XXX: Since Python 2.7.9 ssl cert validation is enabled by default |
243 | # see PEP-0476, this causes verification errors on some https servers | 280 | # see PEP-0476, this causes verification errors on some https servers |
244 | # so disable by default. | 281 | # so disable by default. |
245 | import ssl | 282 | import ssl |
246 | ssl_context = None | ||
247 | if hasattr(ssl, '_create_unverified_context'): | 283 | if hasattr(ssl, '_create_unverified_context'): |
248 | ssl_context = ssl._create_unverified_context() | 284 | handlers.append(urllib2.HTTPSHandler(context=ssl._create_unverified_context())) |
249 | 285 | opener = urllib2.build_opener(*handlers) | |
250 | if exported_proxies == True and ssl_context is not None: | ||
251 | opener = urllib2.build_opener(urllib2.ProxyHandler, CacheHTTPHandler, | ||
252 | urllib2.HTTPSHandler(context=ssl_context)) | ||
253 | elif exported_proxies == False and ssl_context is not None: | ||
254 | opener = urllib2.build_opener(CacheHTTPHandler, | ||
255 | urllib2.HTTPSHandler(context=ssl_context)) | ||
256 | elif exported_proxies == True and ssl_context is None: | ||
257 | opener = urllib2.build_opener(urllib2.ProxyHandler, CacheHTTPHandler) | ||
258 | else: | ||
259 | opener = urllib2.build_opener(CacheHTTPHandler) | ||
260 | |||
261 | urllib2.Request.get_method = head_method | ||
262 | urllib2.install_opener(opener) | ||
263 | |||
264 | uri = ud.url.split(";")[0] | ||
265 | 286 | ||
266 | try: | 287 | try: |
267 | urllib2.urlopen(uri) | 288 | uri = ud.url.split(";")[0] |
268 | except: | 289 | r = urllib2.Request(uri) |
290 | r.get_method = lambda: "HEAD" | ||
291 | opener.open(r) | ||
292 | except urllib2.URLError as e: | ||
293 | # debug for now to avoid spamming the logs in e.g. remote sstate searches | ||
294 | logger.debug(2, "checkstatus() urlopen failed: %s" % e) | ||
269 | return False | 295 | return False |
270 | return True | 296 | return True |
271 | 297 | ||
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py index 7a08be0f49..3b74de0d70 100644 --- a/bitbake/lib/bb/tests/fetch.py +++ b/bitbake/lib/bb/tests/fetch.py | |||
@@ -737,6 +737,8 @@ class FetchCheckStatusTest(FetcherTest): | |||
737 | "ftp://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz", | 737 | "ftp://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz", |
738 | "ftp://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz", | 738 | "ftp://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz", |
739 | "ftp://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz", | 739 | "ftp://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz", |
740 | # GitHub releases are hosted on Amazon S3, which doesn't support HEAD | ||
741 | "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz" | ||
740 | ] | 742 | ] |
741 | 743 | ||
742 | if os.environ.get("BB_SKIP_NETTESTS") == "yes": | 744 | if os.environ.get("BB_SKIP_NETTESTS") == "yes": |