From 45baa90c8cb0f5e92e6a2847d84312b52b0582a2 Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Wed, 6 Jul 2016 16:26:10 +1200 Subject: bitbake: fetch2: implement progress support Implement progress reporting support specifically for the fetchers. For fetch tasks we don't necessarily know which fetcher will be used (we might initially be fetching a git:// URI, but if we instead download a mirror tarball we may fetch that over http using wget). These programs also have different abilities as far as reporting progress goes (e.g. wget gives us percentage complete and rate, git gives this some of the time depending on what stage it's at). Additionally we filter out the progress output before it makes it to the logs, in order to prevent the logs filling up with junk. At the moment this is only implemented for the wget and git fetchers since they are the most commonly used (and svn doesn't seem to support any kind of progress output, at least not without doing a relatively expensive remote file listing first). Line changes such as the ones you get in git's output as it progresses don't make it to the log files, you only get the final state of the line so the logs aren't filled with progress information that's useless after the fact. Part of the implementation for [YOCTO #5383]. (Bitbake rev: 4027649f422ee64b1c4e1ad8d48ac295050afbff) Signed-off-by: Paul Eggleton Signed-off-by: Ross Burton Signed-off-by: Richard Purdie --- bitbake/lib/bb/fetch2/git.py | 52 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) (limited to 'bitbake/lib/bb/fetch2/git.py') diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index 59827e304f..4e2dcec0d7 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py @@ -71,11 +71,53 @@ import os import re import bb import errno +import bb.progress from bb import data from bb.fetch2 import FetchMethod from bb.fetch2 import runfetchcmd from bb.fetch2 import logger + +class GitProgressHandler(bb.progress.LineFilterProgressHandler): + """Extract progress information from git output""" + def __init__(self, d): + self._buffer = '' + self._count = 0 + super(GitProgressHandler, self).__init__(d) + # Send an initial progress event so the bar gets shown + self._fire_progress(-1) + + def write(self, string): + self._buffer += string + stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas'] + stage_weights = [0.2, 0.05, 0.5, 0.25] + stagenum = 0 + for i, stage in reversed(list(enumerate(stages))): + if stage in self._buffer: + stagenum = i + self._buffer = '' + break + self._status = stages[stagenum] + percs = re.findall(r'(\d+)%', string) + if percs: + progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100))) + rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string) + if rates: + rate = rates[-1] + else: + rate = None + self.update(progress, rate) + else: + if stagenum == 0: + percs = re.findall(r': (\d+)', string) + if percs: + count = int(percs[-1]) + if count > self._count: + self._count = count + self._fire_progress(-count) + super(GitProgressHandler, self).write(string) + + class Git(FetchMethod): """Class to fetch a module or modules from git repositories""" def init(self, d): @@ -196,10 +238,11 @@ class Git(FetchMethod): # We do this since git will use a "-l" option automatically for local urls where possible if repourl.startswith("file://"): repourl = repourl[7:] - clone_cmd = "%s clone --bare --mirror %s %s" % (ud.basecmd, repourl, ud.clonedir) + clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, clone_cmd) - runfetchcmd(clone_cmd, d) + progresshandler = GitProgressHandler(d) + runfetchcmd(clone_cmd, d, log=progresshandler) os.chdir(ud.clonedir) # Update the checkout if needed @@ -214,10 +257,11 @@ class Git(FetchMethod): logger.debug(1, "No Origin") runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d) - fetch_cmd = "%s fetch -f --prune %s refs/*:refs/*" % (ud.basecmd, repourl) + fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, fetch_cmd, ud.url) - runfetchcmd(fetch_cmd, d) + progresshandler = GitProgressHandler(d) + runfetchcmd(fetch_cmd, d, log=progresshandler) runfetchcmd("%s prune-packed" % ud.basecmd, d) runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) try: -- cgit v1.2.3-54-g00ecf