From 1a4e4fb6b0a9d54641bd4193e95311d1f822a9ca Mon Sep 17 00:00:00 2001 From: Chris Laplante via bitbake-devel Date: Fri, 26 Jul 2019 13:18:37 -0400 Subject: bitbake: fetch2/wget: avoid 'maximum recursion depth' RuntimeErrors when handling 403 codes The code says that some servers respond with 403 codes when they really mean 405 codes. But we still need to account for legitimate 403 codes. Before this change, I noticed that sstate mirror checking was taking a very long time when I purposely entered incorrect credentials into my .netrc file for our sstate mirror. Instrumenting the code, I discovered tracebacks like the following for every mirror access attempt: File "/home/laplante/yocto/sources/poky/meta/classes/sstate.bbclass", line 839, in checkstatus fetcher.checkstatus() File "/home/laplante/yocto/sources/poky/bitbake/lib/bb/fetch2/__init__.py", line 1736, in checkstatus ret = try_mirrors(self, self.d, ud, mirrors, True) File "/home/laplante/yocto/sources/poky/bitbake/lib/bb/fetch2/__init__.py", line 1077, in try_mirrors ret = try_mirror_url(fetch, origud, uds[index], ld, check) File "/home/laplante/yocto/sources/poky/bitbake/lib/bb/fetch2/__init__.py", line 979, in try_mirror_url found = ud.method.checkstatus(fetch, ud, ld) File "/home/laplante/yocto/sources/poky/bitbake/lib/bb/fetch2/wget.py", line 337, in checkstatus opener.open(r) File "/usr/lib/python3.5/urllib/request.py", line 472, in open response = meth(req, response) File "/usr/lib/python3.5/urllib/request.py", line 582, in http_response 'http', request, response, code, msg, hdrs) File "/usr/lib/python3.5/urllib/request.py", line 504, in error result = self._call_chain(*args) File "/usr/lib/python3.5/urllib/request.py", line 444, in _call_chain result = func(*args) File "/home/laplante/yocto/sources/poky/bitbake/lib/bb/fetch2/wget.py", line 280, in http_error_405 unverifiable=True)) File "/usr/lib/python3.5/urllib/request.py", line 472, in open response = meth(req, response) File "/usr/lib/python3.5/urllib/request.py", line 582, in http_response 'http', request, response, code, msg, hdrs) File "/usr/lib/python3.5/urllib/request.py", line 504, in error result = self._call_chain(*args) File "/usr/lib/python3.5/urllib/request.py", line 444, in _call_chain result = func(*args) File "/home/laplante/yocto/sources/poky/bitbake/lib/bb/fetch2/wget.py", line 280, in http_error_405 unverifiable=True)) ... (repeats until recursion depth is reached) Solution is to make sure we only attempt the GET request once when handling 403/405 error codes. (Bitbake rev: 18d4a31fdcec1f0e5d2199d6142f0ce833fca1a7) Signed-off-by: Chris Laplante Signed-off-by: Richard Purdie --- bitbake/lib/bb/fetch2/wget.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'bitbake/lib/bb/fetch2') diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py index 0f71ee4eac..725586d2b5 100644 --- a/bitbake/lib/bb/fetch2/wget.py +++ b/bitbake/lib/bb/fetch2/wget.py @@ -257,13 +257,15 @@ class Wget(FetchMethod): fp.read() fp.close() - newheaders = dict((k, v) for k, v in list(req.headers.items()) - if k.lower() not in ("content-length", "content-type")) - return self.parent.open(urllib.request.Request(req.get_full_url(), - headers=newheaders, - origin_req_host=req.origin_req_host, - unverifiable=True)) - + if req.get_method() != 'GET': + newheaders = dict((k, v) for k, v in list(req.headers.items()) + if k.lower() not in ("content-length", "content-type")) + return self.parent.open(urllib.request.Request(req.get_full_url(), + headers=newheaders, + origin_req_host=req.origin_req_host, + unverifiable=True)) + + raise urllib.request.HTTPError(req, code, msg, headers, None) # Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403 # Forbidden when they actually mean 405 Method Not Allowed. -- cgit v1.2.3-54-g00ecf