diff options
Diffstat (limited to 'bitbake/lib/bb/fetch2/wget.py')
-rw-r--r-- | bitbake/lib/bb/fetch2/wget.py | 24 |
1 files changed, 16 insertions, 8 deletions
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py index f7d1de26b7..368c644337 100644 --- a/bitbake/lib/bb/fetch2/wget.py +++ b/bitbake/lib/bb/fetch2/wget.py | |||
@@ -52,6 +52,12 @@ class WgetProgressHandler(bb.progress.LineFilterProgressHandler): | |||
52 | 52 | ||
53 | 53 | ||
54 | class Wget(FetchMethod): | 54 | class Wget(FetchMethod): |
55 | |||
56 | # CDNs like CloudFlare may do a 'browser integrity test' which can fail | ||
57 | # with the standard wget/urllib User-Agent, so pretend to be a modern | ||
58 | # browser. | ||
59 | user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0" | ||
60 | |||
55 | """Class to fetch urls via 'wget'""" | 61 | """Class to fetch urls via 'wget'""" |
56 | def supports(self, ud, d): | 62 | def supports(self, ud, d): |
57 | """ | 63 | """ |
@@ -91,10 +97,9 @@ class Wget(FetchMethod): | |||
91 | 97 | ||
92 | fetchcmd = self.basecmd | 98 | fetchcmd = self.basecmd |
93 | 99 | ||
94 | if 'downloadfilename' in ud.parm: | 100 | localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) + ".tmp" |
95 | localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) | 101 | bb.utils.mkdirhier(os.path.dirname(localpath)) |
96 | bb.utils.mkdirhier(os.path.dirname(localpath)) | 102 | fetchcmd += " -O %s" % shlex.quote(localpath) |
97 | fetchcmd += " -O %s" % shlex.quote(localpath) | ||
98 | 103 | ||
99 | if ud.user and ud.pswd: | 104 | if ud.user and ud.pswd: |
100 | fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd) | 105 | fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd) |
@@ -108,6 +113,10 @@ class Wget(FetchMethod): | |||
108 | 113 | ||
109 | self._runwget(ud, d, fetchcmd, False) | 114 | self._runwget(ud, d, fetchcmd, False) |
110 | 115 | ||
116 | # Remove the ".tmp" and move the file into position atomically | ||
117 | # Our lock prevents multiple writers but mirroring code may grab incomplete files | ||
118 | os.rename(localpath, localpath[:-4]) | ||
119 | |||
111 | # Sanity check since wget can pretend it succeed when it didn't | 120 | # Sanity check since wget can pretend it succeed when it didn't |
112 | # Also, this used to happen if sourceforge sent us to the mirror page | 121 | # Also, this used to happen if sourceforge sent us to the mirror page |
113 | if not os.path.exists(ud.localpath): | 122 | if not os.path.exists(ud.localpath): |
@@ -300,7 +309,7 @@ class Wget(FetchMethod): | |||
300 | # Some servers (FusionForge, as used on Alioth) require that the | 309 | # Some servers (FusionForge, as used on Alioth) require that the |
301 | # optional Accept header is set. | 310 | # optional Accept header is set. |
302 | r.add_header("Accept", "*/*") | 311 | r.add_header("Accept", "*/*") |
303 | r.add_header("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12") | 312 | r.add_header("User-Agent", self.user_agent) |
304 | def add_basic_auth(login_str, request): | 313 | def add_basic_auth(login_str, request): |
305 | '''Adds Basic auth to http request, pass in login:password as string''' | 314 | '''Adds Basic auth to http request, pass in login:password as string''' |
306 | import base64 | 315 | import base64 |
@@ -319,7 +328,7 @@ class Wget(FetchMethod): | |||
319 | except (TypeError, ImportError, IOError, netrc.NetrcParseError): | 328 | except (TypeError, ImportError, IOError, netrc.NetrcParseError): |
320 | pass | 329 | pass |
321 | 330 | ||
322 | with opener.open(r) as response: | 331 | with opener.open(r, timeout=30) as response: |
323 | pass | 332 | pass |
324 | except urllib.error.URLError as e: | 333 | except urllib.error.URLError as e: |
325 | if try_again: | 334 | if try_again: |
@@ -404,9 +413,8 @@ class Wget(FetchMethod): | |||
404 | """ | 413 | """ |
405 | f = tempfile.NamedTemporaryFile() | 414 | f = tempfile.NamedTemporaryFile() |
406 | with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f: | 415 | with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f: |
407 | agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12" | ||
408 | fetchcmd = self.basecmd | 416 | fetchcmd = self.basecmd |
409 | fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'" | 417 | fetchcmd += " -O " + f.name + " --user-agent='" + self.user_agent + "' '" + uri + "'" |
410 | try: | 418 | try: |
411 | self._runwget(ud, d, fetchcmd, True, workdir=workdir) | 419 | self._runwget(ud, d, fetchcmd, True, workdir=workdir) |
412 | fetchresult = f.read() | 420 | fetchresult = f.read() |