summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/fetch2
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2016-07-06 16:26:10 +1200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-07-19 15:04:37 +0100
commit45baa90c8cb0f5e92e6a2847d84312b52b0582a2 (patch)
tree1264d6664a4d8fc710838237c3e842ff54fd1c76 /bitbake/lib/bb/fetch2
parent843f7ae4108ad862b02ae84eef4f8cf7f6a3c950 (diff)
downloadpoky-45baa90c8cb0f5e92e6a2847d84312b52b0582a2.tar.gz
bitbake: fetch2: implement progress support
Implement progress reporting support specifically for the fetchers. For fetch tasks we don't necessarily know which fetcher will be used (we might initially be fetching a git:// URI, but if we instead download a mirror tarball we may fetch that over http using wget). These programs also have different abilities as far as reporting progress goes (e.g. wget gives us percentage complete and rate, git gives this some of the time depending on what stage it's at). Additionally we filter out the progress output before it makes it to the logs, in order to prevent the logs filling up with junk. At the moment this is only implemented for the wget and git fetchers since they are the most commonly used (and svn doesn't seem to support any kind of progress output, at least not without doing a relatively expensive remote file listing first). Line changes such as the ones you get in git's output as it progresses don't make it to the log files, you only get the final state of the line so the logs aren't filled with progress information that's useless after the fact. Part of the implementation for [YOCTO #5383]. (Bitbake rev: 4027649f422ee64b1c4e1ad8d48ac295050afbff) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/fetch2')
-rw-r--r--bitbake/lib/bb/fetch2/__init__.py4
-rw-r--r--bitbake/lib/bb/fetch2/git.py52
-rw-r--r--bitbake/lib/bb/fetch2/wget.py28
3 files changed, 76 insertions, 8 deletions
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index b6fcaaad55..a27512cc87 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -779,7 +779,7 @@ def localpath(url, d):
779 fetcher = bb.fetch2.Fetch([url], d) 779 fetcher = bb.fetch2.Fetch([url], d)
780 return fetcher.localpath(url) 780 return fetcher.localpath(url)
781 781
782def runfetchcmd(cmd, d, quiet=False, cleanup=None): 782def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None):
783 """ 783 """
784 Run cmd returning the command output 784 Run cmd returning the command output
785 Raise an error if interrupted or cmd fails 785 Raise an error if interrupted or cmd fails
@@ -821,7 +821,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None):
821 error_message = "" 821 error_message = ""
822 822
823 try: 823 try:
824 (output, errors) = bb.process.run(cmd, shell=True, stderr=subprocess.PIPE) 824 (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE)
825 success = True 825 success = True
826 except bb.process.NotFoundError as e: 826 except bb.process.NotFoundError as e:
827 error_message = "Fetch command %s" % (e.command) 827 error_message = "Fetch command %s" % (e.command)
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 59827e304f..4e2dcec0d7 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -71,11 +71,53 @@ import os
71import re 71import re
72import bb 72import bb
73import errno 73import errno
74import bb.progress
74from bb import data 75from bb import data
75from bb.fetch2 import FetchMethod 76from bb.fetch2 import FetchMethod
76from bb.fetch2 import runfetchcmd 77from bb.fetch2 import runfetchcmd
77from bb.fetch2 import logger 78from bb.fetch2 import logger
78 79
80
81class GitProgressHandler(bb.progress.LineFilterProgressHandler):
82 """Extract progress information from git output"""
83 def __init__(self, d):
84 self._buffer = ''
85 self._count = 0
86 super(GitProgressHandler, self).__init__(d)
87 # Send an initial progress event so the bar gets shown
88 self._fire_progress(-1)
89
90 def write(self, string):
91 self._buffer += string
92 stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas']
93 stage_weights = [0.2, 0.05, 0.5, 0.25]
94 stagenum = 0
95 for i, stage in reversed(list(enumerate(stages))):
96 if stage in self._buffer:
97 stagenum = i
98 self._buffer = ''
99 break
100 self._status = stages[stagenum]
101 percs = re.findall(r'(\d+)%', string)
102 if percs:
103 progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100)))
104 rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string)
105 if rates:
106 rate = rates[-1]
107 else:
108 rate = None
109 self.update(progress, rate)
110 else:
111 if stagenum == 0:
112 percs = re.findall(r': (\d+)', string)
113 if percs:
114 count = int(percs[-1])
115 if count > self._count:
116 self._count = count
117 self._fire_progress(-count)
118 super(GitProgressHandler, self).write(string)
119
120
79class Git(FetchMethod): 121class Git(FetchMethod):
80 """Class to fetch a module or modules from git repositories""" 122 """Class to fetch a module or modules from git repositories"""
81 def init(self, d): 123 def init(self, d):
@@ -196,10 +238,11 @@ class Git(FetchMethod):
196 # We do this since git will use a "-l" option automatically for local urls where possible 238 # We do this since git will use a "-l" option automatically for local urls where possible
197 if repourl.startswith("file://"): 239 if repourl.startswith("file://"):
198 repourl = repourl[7:] 240 repourl = repourl[7:]
199 clone_cmd = "%s clone --bare --mirror %s %s" % (ud.basecmd, repourl, ud.clonedir) 241 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir)
200 if ud.proto.lower() != 'file': 242 if ud.proto.lower() != 'file':
201 bb.fetch2.check_network_access(d, clone_cmd) 243 bb.fetch2.check_network_access(d, clone_cmd)
202 runfetchcmd(clone_cmd, d) 244 progresshandler = GitProgressHandler(d)
245 runfetchcmd(clone_cmd, d, log=progresshandler)
203 246
204 os.chdir(ud.clonedir) 247 os.chdir(ud.clonedir)
205 # Update the checkout if needed 248 # Update the checkout if needed
@@ -214,10 +257,11 @@ class Git(FetchMethod):
214 logger.debug(1, "No Origin") 257 logger.debug(1, "No Origin")
215 258
216 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d) 259 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d)
217 fetch_cmd = "%s fetch -f --prune %s refs/*:refs/*" % (ud.basecmd, repourl) 260 fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl)
218 if ud.proto.lower() != 'file': 261 if ud.proto.lower() != 'file':
219 bb.fetch2.check_network_access(d, fetch_cmd, ud.url) 262 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
220 runfetchcmd(fetch_cmd, d) 263 progresshandler = GitProgressHandler(d)
264 runfetchcmd(fetch_cmd, d, log=progresshandler)
221 runfetchcmd("%s prune-packed" % ud.basecmd, d) 265 runfetchcmd("%s prune-packed" % ud.basecmd, d)
222 runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) 266 runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
223 try: 267 try:
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index 6cb22aeee9..6b60d9b16b 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -31,6 +31,7 @@ import subprocess
31import os 31import os
32import logging 32import logging
33import bb 33import bb
34import bb.progress
34import urllib.request, urllib.parse, urllib.error 35import urllib.request, urllib.parse, urllib.error
35from bb import data 36from bb import data
36from bb.fetch2 import FetchMethod 37from bb.fetch2 import FetchMethod
@@ -41,6 +42,27 @@ from bb.utils import export_proxies
41from bs4 import BeautifulSoup 42from bs4 import BeautifulSoup
42from bs4 import SoupStrainer 43from bs4 import SoupStrainer
43 44
45class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
46 """
47 Extract progress information from wget output.
48 Note: relies on --progress=dot (with -v or without -q/-nv) being
49 specified on the wget command line.
50 """
51 def __init__(self, d):
52 super(WgetProgressHandler, self).__init__(d)
53 # Send an initial progress event so the bar gets shown
54 self._fire_progress(0)
55
56 def writeline(self, line):
57 percs = re.findall(r'(\d+)%\s+([\d.]+[A-Z])', line)
58 if percs:
59 progress = int(percs[-1][0])
60 rate = percs[-1][1] + '/s'
61 self.update(progress, rate)
62 return False
63 return True
64
65
44class Wget(FetchMethod): 66class Wget(FetchMethod):
45 """Class to fetch urls via 'wget'""" 67 """Class to fetch urls via 'wget'"""
46 def supports(self, ud, d): 68 def supports(self, ud, d):
@@ -66,13 +88,15 @@ class Wget(FetchMethod):
66 if not ud.localfile: 88 if not ud.localfile:
67 ud.localfile = data.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."), d) 89 ud.localfile = data.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."), d)
68 90
69 self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 -nv --passive-ftp --no-check-certificate" 91 self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
70 92
71 def _runwget(self, ud, d, command, quiet): 93 def _runwget(self, ud, d, command, quiet):
72 94
95 progresshandler = WgetProgressHandler(d)
96
73 logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command)) 97 logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
74 bb.fetch2.check_network_access(d, command) 98 bb.fetch2.check_network_access(d, command)
75 runfetchcmd(command, d, quiet) 99 runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler)
76 100
77 def download(self, ud, d): 101 def download(self, ud, d):
78 """Fetch urls""" 102 """Fetch urls"""