summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/fetch2/wget.py
diff options
context:
space:
mode:
authorRoss Burton <ross.burton@intel.com>2017-07-24 22:42:01 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-07-25 15:53:19 +0100
commit3c1fba5d30616c1ab793c45f169c8f44310ab435 (patch)
tree489b109d7f1f01913eb41828e1f6d75661eb6096 /bitbake/lib/bb/fetch2/wget.py
parent33b73b2d791716485f48ff56b57ce03b05174410 (diff)
downloadpoky-3c1fba5d30616c1ab793c45f169c8f44310ab435.tar.gz
bitbake: fetch/wget: mitigate a wget race condition when listing FTP directories
When wget is fetching a listing for a directory over FTP it writes to a temporary file called .listing in the current directory. If there are many such operations happening in parallel - for example during 'bitbake world -c checkpkg' - then up to BB_NUMBER_THREADS instances of wget will be racing to write to, read, and delete the same file. This results in various failures such as the file disappearing before wget has processed it or the file changing contents, which causes checkpkg to randomly fail. Mitigate the race condition by creating a temporary directory to run wget in when doing directory listings. [ YOCTO #11828 ] (Bitbake rev: 91d4ca93df092cf86ab84faaa94cc66ff9f43057) Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/fetch2/wget.py')
-rw-r--r--bitbake/lib/bb/fetch2/wget.py23
1 files changed, 11 insertions, 12 deletions
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index 208ee9bdd6..8ee9769d39 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -90,13 +90,13 @@ class Wget(FetchMethod):
90 90
91 self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate" 91 self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
92 92
93 def _runwget(self, ud, d, command, quiet): 93 def _runwget(self, ud, d, command, quiet, workdir=None):
94 94
95 progresshandler = WgetProgressHandler(d) 95 progresshandler = WgetProgressHandler(d)
96 96
97 logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command)) 97 logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
98 bb.fetch2.check_network_access(d, command, ud.url) 98 bb.fetch2.check_network_access(d, command, ud.url)
99 runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler) 99 runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir)
100 100
101 def download(self, ud, d): 101 def download(self, ud, d):
102 """Fetch urls""" 102 """Fetch urls"""
@@ -422,17 +422,16 @@ class Wget(FetchMethod):
422 Run fetch checkstatus to get directory information 422 Run fetch checkstatus to get directory information
423 """ 423 """
424 f = tempfile.NamedTemporaryFile() 424 f = tempfile.NamedTemporaryFile()
425 with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f:
426 agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12"
427 fetchcmd = self.basecmd
428 fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'"
429 try:
430 self._runwget(ud, d, fetchcmd, True, workdir=workdir)
431 fetchresult = f.read()
432 except bb.fetch2.BBFetchException:
433 fetchresult = ""
425 434
426 agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12"
427 fetchcmd = self.basecmd
428 fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'"
429 try:
430 self._runwget(ud, d, fetchcmd, True)
431 fetchresult = f.read()
432 except bb.fetch2.BBFetchException:
433 fetchresult = ""
434
435 f.close()
436 return fetchresult 435 return fetchresult
437 436
438 def _check_latest_version(self, url, package, package_regex, current_version, ud, d): 437 def _check_latest_version(self, url, package, package_regex, current_version, ud, d):