summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2016-07-06 16:26:10 +1200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-07-19 15:04:37 +0100
commit45baa90c8cb0f5e92e6a2847d84312b52b0582a2 (patch)
tree1264d6664a4d8fc710838237c3e842ff54fd1c76
parent843f7ae4108ad862b02ae84eef4f8cf7f6a3c950 (diff)
downloadpoky-45baa90c8cb0f5e92e6a2847d84312b52b0582a2.tar.gz
bitbake: fetch2: implement progress support
Implement progress reporting support specifically for the fetchers. For fetch tasks we don't necessarily know which fetcher will be used (we might initially be fetching a git:// URI, but if we instead download a mirror tarball we may fetch that over http using wget). These programs also have different abilities as far as reporting progress goes (e.g. wget gives us percentage complete and rate, git gives this some of the time depending on what stage it's at). Additionally we filter out the progress output before it makes it to the logs, in order to prevent the logs filling up with junk. At the moment this is only implemented for the wget and git fetchers since they are the most commonly used (and svn doesn't seem to support any kind of progress output, at least not without doing a relatively expensive remote file listing first). Line changes such as the ones you get in git's output as it progresses don't make it to the log files, you only get the final state of the line so the logs aren't filled with progress information that's useless after the fact. Part of the implementation for [YOCTO #5383]. (Bitbake rev: 4027649f422ee64b1c4e1ad8d48ac295050afbff) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/fetch2/__init__.py4
-rw-r--r--bitbake/lib/bb/fetch2/git.py52
-rw-r--r--bitbake/lib/bb/fetch2/wget.py28
-rw-r--r--bitbake/lib/bb/progress.py31
4 files changed, 107 insertions, 8 deletions
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index b6fcaaad55..a27512cc87 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -779,7 +779,7 @@ def localpath(url, d):
779 fetcher = bb.fetch2.Fetch([url], d) 779 fetcher = bb.fetch2.Fetch([url], d)
780 return fetcher.localpath(url) 780 return fetcher.localpath(url)
781 781
782def runfetchcmd(cmd, d, quiet=False, cleanup=None): 782def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None):
783 """ 783 """
784 Run cmd returning the command output 784 Run cmd returning the command output
785 Raise an error if interrupted or cmd fails 785 Raise an error if interrupted or cmd fails
@@ -821,7 +821,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None):
821 error_message = "" 821 error_message = ""
822 822
823 try: 823 try:
824 (output, errors) = bb.process.run(cmd, shell=True, stderr=subprocess.PIPE) 824 (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE)
825 success = True 825 success = True
826 except bb.process.NotFoundError as e: 826 except bb.process.NotFoundError as e:
827 error_message = "Fetch command %s" % (e.command) 827 error_message = "Fetch command %s" % (e.command)
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 59827e304f..4e2dcec0d7 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -71,11 +71,53 @@ import os
71import re 71import re
72import bb 72import bb
73import errno 73import errno
74import bb.progress
74from bb import data 75from bb import data
75from bb.fetch2 import FetchMethod 76from bb.fetch2 import FetchMethod
76from bb.fetch2 import runfetchcmd 77from bb.fetch2 import runfetchcmd
77from bb.fetch2 import logger 78from bb.fetch2 import logger
78 79
80
81class GitProgressHandler(bb.progress.LineFilterProgressHandler):
82 """Extract progress information from git output"""
83 def __init__(self, d):
84 self._buffer = ''
85 self._count = 0
86 super(GitProgressHandler, self).__init__(d)
87 # Send an initial progress event so the bar gets shown
88 self._fire_progress(-1)
89
90 def write(self, string):
91 self._buffer += string
92 stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas']
93 stage_weights = [0.2, 0.05, 0.5, 0.25]
94 stagenum = 0
95 for i, stage in reversed(list(enumerate(stages))):
96 if stage in self._buffer:
97 stagenum = i
98 self._buffer = ''
99 break
100 self._status = stages[stagenum]
101 percs = re.findall(r'(\d+)%', string)
102 if percs:
103 progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100)))
104 rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string)
105 if rates:
106 rate = rates[-1]
107 else:
108 rate = None
109 self.update(progress, rate)
110 else:
111 if stagenum == 0:
112 percs = re.findall(r': (\d+)', string)
113 if percs:
114 count = int(percs[-1])
115 if count > self._count:
116 self._count = count
117 self._fire_progress(-count)
118 super(GitProgressHandler, self).write(string)
119
120
79class Git(FetchMethod): 121class Git(FetchMethod):
80 """Class to fetch a module or modules from git repositories""" 122 """Class to fetch a module or modules from git repositories"""
81 def init(self, d): 123 def init(self, d):
@@ -196,10 +238,11 @@ class Git(FetchMethod):
196 # We do this since git will use a "-l" option automatically for local urls where possible 238 # We do this since git will use a "-l" option automatically for local urls where possible
197 if repourl.startswith("file://"): 239 if repourl.startswith("file://"):
198 repourl = repourl[7:] 240 repourl = repourl[7:]
199 clone_cmd = "%s clone --bare --mirror %s %s" % (ud.basecmd, repourl, ud.clonedir) 241 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir)
200 if ud.proto.lower() != 'file': 242 if ud.proto.lower() != 'file':
201 bb.fetch2.check_network_access(d, clone_cmd) 243 bb.fetch2.check_network_access(d, clone_cmd)
202 runfetchcmd(clone_cmd, d) 244 progresshandler = GitProgressHandler(d)
245 runfetchcmd(clone_cmd, d, log=progresshandler)
203 246
204 os.chdir(ud.clonedir) 247 os.chdir(ud.clonedir)
205 # Update the checkout if needed 248 # Update the checkout if needed
@@ -214,10 +257,11 @@ class Git(FetchMethod):
214 logger.debug(1, "No Origin") 257 logger.debug(1, "No Origin")
215 258
216 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d) 259 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d)
217 fetch_cmd = "%s fetch -f --prune %s refs/*:refs/*" % (ud.basecmd, repourl) 260 fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl)
218 if ud.proto.lower() != 'file': 261 if ud.proto.lower() != 'file':
219 bb.fetch2.check_network_access(d, fetch_cmd, ud.url) 262 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
220 runfetchcmd(fetch_cmd, d) 263 progresshandler = GitProgressHandler(d)
264 runfetchcmd(fetch_cmd, d, log=progresshandler)
221 runfetchcmd("%s prune-packed" % ud.basecmd, d) 265 runfetchcmd("%s prune-packed" % ud.basecmd, d)
222 runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) 266 runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
223 try: 267 try:
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index 6cb22aeee9..6b60d9b16b 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -31,6 +31,7 @@ import subprocess
31import os 31import os
32import logging 32import logging
33import bb 33import bb
34import bb.progress
34import urllib.request, urllib.parse, urllib.error 35import urllib.request, urllib.parse, urllib.error
35from bb import data 36from bb import data
36from bb.fetch2 import FetchMethod 37from bb.fetch2 import FetchMethod
@@ -41,6 +42,27 @@ from bb.utils import export_proxies
41from bs4 import BeautifulSoup 42from bs4 import BeautifulSoup
42from bs4 import SoupStrainer 43from bs4 import SoupStrainer
43 44
45class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
46 """
47 Extract progress information from wget output.
48 Note: relies on --progress=dot (with -v or without -q/-nv) being
49 specified on the wget command line.
50 """
51 def __init__(self, d):
52 super(WgetProgressHandler, self).__init__(d)
53 # Send an initial progress event so the bar gets shown
54 self._fire_progress(0)
55
56 def writeline(self, line):
57 percs = re.findall(r'(\d+)%\s+([\d.]+[A-Z])', line)
58 if percs:
59 progress = int(percs[-1][0])
60 rate = percs[-1][1] + '/s'
61 self.update(progress, rate)
62 return False
63 return True
64
65
44class Wget(FetchMethod): 66class Wget(FetchMethod):
45 """Class to fetch urls via 'wget'""" 67 """Class to fetch urls via 'wget'"""
46 def supports(self, ud, d): 68 def supports(self, ud, d):
@@ -66,13 +88,15 @@ class Wget(FetchMethod):
66 if not ud.localfile: 88 if not ud.localfile:
67 ud.localfile = data.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."), d) 89 ud.localfile = data.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."), d)
68 90
69 self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 -nv --passive-ftp --no-check-certificate" 91 self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
70 92
71 def _runwget(self, ud, d, command, quiet): 93 def _runwget(self, ud, d, command, quiet):
72 94
95 progresshandler = WgetProgressHandler(d)
96
73 logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command)) 97 logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
74 bb.fetch2.check_network_access(d, command) 98 bb.fetch2.check_network_access(d, command)
75 runfetchcmd(command, d, quiet) 99 runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler)
76 100
77 def download(self, ud, d): 101 def download(self, ud, d):
78 """Fetch urls""" 102 """Fetch urls"""
diff --git a/bitbake/lib/bb/progress.py b/bitbake/lib/bb/progress.py
index ee6b9536b8..343b18f8c4 100644
--- a/bitbake/lib/bb/progress.py
+++ b/bitbake/lib/bb/progress.py
@@ -58,6 +58,37 @@ class ProgressHandler(object):
58 self._lastevent = ts 58 self._lastevent = ts
59 self._progress = progress 59 self._progress = progress
60 60
61class LineFilterProgressHandler(ProgressHandler):
62 """
63 A ProgressHandler variant that provides the ability to filter out
64 the lines if they contain progress information. Additionally, it
65 filters out anything before the last line feed on a line. This can
66 be used to keep the logs clean of output that we've only enabled for
67 getting progress, assuming that that can be done on a per-line
68 basis.
69 """
70 def __init__(self, d, outfile=None):
71 self._linebuffer = ''
72 super(LineFilterProgressHandler, self).__init__(d, outfile)
73
74 def write(self, string):
75 self._linebuffer += string
76 while True:
77 breakpos = self._linebuffer.find('\n') + 1
78 if breakpos == 0:
79 break
80 line = self._linebuffer[:breakpos]
81 self._linebuffer = self._linebuffer[breakpos:]
82 # Drop any line feeds and anything that precedes them
83 lbreakpos = line.rfind('\r') + 1
84 if lbreakpos:
85 line = line[lbreakpos:]
86 if self.writeline(line):
87 super(LineFilterProgressHandler, self).write(line)
88
89 def writeline(self, line):
90 return True
91
61class BasicProgressHandler(ProgressHandler): 92class BasicProgressHandler(ProgressHandler):
62 def __init__(self, d, regex=r'(\d+)%', outfile=None): 93 def __init__(self, d, regex=r'(\d+)%', outfile=None):
63 super(BasicProgressHandler, self).__init__(d, outfile) 94 super(BasicProgressHandler, self).__init__(d, outfile)