summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorRichard Purdie <richard@openedhand.com>2007-04-01 15:04:49 +0000
committerRichard Purdie <richard@openedhand.com>2007-04-01 15:04:49 +0000
commit7371e6323c3fb6b0545712e3cf84606644073e77 (patch)
treee08f25669ec0f0e9d11334909f3b68c0ab6aca19 /bitbake
parent8b36dc217443aeeec8493d39561d2bb010336774 (diff)
downloadpoky-7371e6323c3fb6b0545712e3cf84606644073e77.tar.gz
bitbake: Update to 1.8.1 (inc. various bug fixes, epoch support)
git-svn-id: https://svn.o-hand.com/repos/poky/trunk@1419 311d38ba-8fff-0310-9ca6-ca027cbcb966
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/ChangeLog54
-rw-r--r--bitbake/MANIFEST26
-rwxr-xr-xbitbake/bin/bitbake14
-rw-r--r--bitbake/classes/base.bbclass79
-rw-r--r--bitbake/conf/bitbake.conf58
-rw-r--r--bitbake/lib/bb/__init__.py2
-rw-r--r--bitbake/lib/bb/build.py3
-rw-r--r--bitbake/lib/bb/cache.py10
-rw-r--r--bitbake/lib/bb/cooker.py318
-rw-r--r--bitbake/lib/bb/event.py5
-rw-r--r--bitbake/lib/bb/fetch/svn.py6
-rw-r--r--bitbake/lib/bb/msg.py31
-rw-r--r--bitbake/lib/bb/parse/parse_py/ConfHandler.py16
-rw-r--r--bitbake/lib/bb/providers.py47
-rw-r--r--bitbake/lib/bb/runqueue.py355
-rw-r--r--bitbake/lib/bb/shell.py25
-rw-r--r--bitbake/lib/bb/taskdata.py13
-rw-r--r--bitbake/lib/bb/utils.py8
18 files changed, 532 insertions, 538 deletions
diff --git a/bitbake/ChangeLog b/bitbake/ChangeLog
index 135aba9fee..0cb85db870 100644
--- a/bitbake/ChangeLog
+++ b/bitbake/ChangeLog
@@ -1,10 +1,52 @@
1Changes in BitBake 1.7.3: 1Changes in Bitbake 1.8.2:
2 - Catch truncated cache file errors
3 - Add PE (Package Epoch) support from Philipp Zabel (pH5)
4 - Add code to handle inter-task dependencies
5 - Allow operations other than assignment on flag variables
2 6
3Changes in BitBake 1.7.1: 7Changes in Bitbake 1.8.0:
4 - Major updates of the dependency handling and execution 8 - Release 1.7.x as a stable series
5 of tasks 9
6 - Change of the SVN Fetcher to keep the checkout around 10Changes in BitBake 1.7.x:
7 courtsey to Paul Sokolovsky (#1367) 11 - Major updates of the dependency handling and execution
12 of tasks. Code from bin/bitbake replaced with runqueue.py
13 and taskdata.py
14 - New task execution code supports multithreading with a simplistic
15 threading algorithm controlled by BB_NUMBER_THREADS
16 - Change of the SVN Fetcher to keep the checkout around
17 courtsey of Paul Sokolovsky (#1367)
18 - PATH fix to bbimage (#1108)
19 - Allow debug domains to be specified on the commandline (-l)
20 - Allow 'interactive' tasks
21 - Logging message improvements
22 - Drop now uneeded BUILD_ALL_DEPS variable
23 - Add support for wildcards to -b option
24 - Major overhaul of the fetchers making a large amount of code common
25 including mirroring code
26 - Fetchers now touch md5 stamps upon access (to show activity)
27 - Fix -f force option when used without -b (long standing bug)
28 - Add expand_cache to data_cache.py, caching expanded data (speedup)
29 - Allow version field in DEPENDS (ignored for now)
30 - Add abort flag support to the shell
31 - Make inherit fail if the class doesn't exist (#1478)
32 - Fix data.emit_env() to expand keynames as well as values
33 - Add ssh fetcher
34 - Add perforce fetcher
35 - Make PREFERRED_PROVIDER_foobar defaults to foobar if available
36 - Share the parser's mtime_cache, reducing the number of stat syscalls
37 - Compile all anonfuncs at once!
38 *** Anonfuncs must now use common spacing format ***
39 - Memorise the list of handlers in __BBHANDLERS and tasks in __BBTASKS
40 This removes 2 million function calls resulting in a 5-10% speedup
41 - Add manpage
42 - Update generateDotGraph to use taskData/runQueue improving accuracy
43 and also adding a task dependency graph
44 - Fix/standardise on GPLv2 licence
45 - Move most functionality from bin/bitbake to cooker.py and split into
46 separate funcitons
47 - CVS fetcher: Added support for non-default port
48 - Add BBINCLUDELOGS_LINES, the number of lines to read from any logfile
49 - Drop shebangs from lib/bb scripts
8 50
9Changes in Bitbake 1.6.0: 51Changes in Bitbake 1.6.0:
10 - Better msg handling 52 - Better msg handling
diff --git a/bitbake/MANIFEST b/bitbake/MANIFEST
index 0440a4e2ea..a0e39c7715 100644
--- a/bitbake/MANIFEST
+++ b/bitbake/MANIFEST
@@ -1,45 +1,49 @@
1AUTHORS 1AUTHORS
2COPYING
2ChangeLog 3ChangeLog
3MANIFEST 4MANIFEST
4setup.py 5setup.py
5bin/bitdoc 6bin/bitdoc
6bin/bbimage 7bin/bbimage
7bin/bitbake 8bin/bitbake
8lib/bb/COW.py
9lib/bb/__init__.py 9lib/bb/__init__.py
10lib/bb/build.py 10lib/bb/build.py
11lib/bb/cache.py 11lib/bb/cache.py
12lib/bb/cooker.py 12lib/bb/cooker.py
13lib/bb/COW.py
13lib/bb/data.py 14lib/bb/data.py
14lib/bb/data_smart.py 15lib/bb/data_smart.py
15lib/bb/event.py 16lib/bb/event.py
16lib/bb/manifest.py 17lib/bb/fetch/__init__.py
17lib/bb/methodpool.py
18lib/bb/msg.py
19lib/bb/providers.py
20lib/bb/runqueue.py
21lib/bb/shell.py
22lib/bb/taskdata.py
23lib/bb/utils.py
24lib/bb/fetch/cvs.py 18lib/bb/fetch/cvs.py
25lib/bb/fetch/git.py 19lib/bb/fetch/git.py
26lib/bb/fetch/__init__.py
27lib/bb/fetch/local.py 20lib/bb/fetch/local.py
28lib/bb/fetch/perforce.py 21lib/bb/fetch/perforce.py
29lib/bb/fetch/ssh.py 22lib/bb/fetch/ssh.py
30lib/bb/fetch/svk.py 23lib/bb/fetch/svk.py
31lib/bb/fetch/svn.py 24lib/bb/fetch/svn.py
32lib/bb/fetch/wget.py 25lib/bb/fetch/wget.py
26lib/bb/manifest.py
27lib/bb/methodpool.py
28lib/bb/msg.py
33lib/bb/parse/__init__.py 29lib/bb/parse/__init__.py
30lib/bb/parse/parse_py/__init__.py
34lib/bb/parse/parse_py/BBHandler.py 31lib/bb/parse/parse_py/BBHandler.py
35lib/bb/parse/parse_py/ConfHandler.py 32lib/bb/parse/parse_py/ConfHandler.py
36lib/bb/parse/parse_py/__init__.py 33lib/bb/providers.py
34lib/bb/runqueue.py
35lib/bb/shell.py
36lib/bb/taskdata.py
37lib/bb/utils.py
38setup.py
37doc/COPYING.GPL 39doc/COPYING.GPL
38doc/COPYING.MIT 40doc/COPYING.MIT
41doc/bitbake.1
39doc/manual/html.css 42doc/manual/html.css
40doc/manual/Makefile 43doc/manual/Makefile
41doc/manual/usermanual.xml 44doc/manual/usermanual.xml
42contrib/bbdev.sh 45contrib/bbdev.sh
43contrib/vim/syntax/bitbake.vim 46contrib/vim/syntax/bitbake.vim
47contrib/vim/ftdetect/bitbake.vim
44conf/bitbake.conf 48conf/bitbake.conf
45classes/base.bbclass 49classes/base.bbclass
diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake
index 001f229331..70a862a815 100755
--- a/bitbake/bin/bitbake
+++ b/bitbake/bin/bitbake
@@ -27,7 +27,7 @@ sys.path.insert(0,os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'l
27import bb 27import bb
28from bb import cooker 28from bb import cooker
29 29
30__version__ = "1.7.4" 30__version__ = "1.8.1"
31 31
32#============================================================================# 32#============================================================================#
33# BBOptions 33# BBOptions
@@ -109,15 +109,9 @@ Default BBFILES are the .bb files in the current directory.""" )
109 configuration.pkgs_to_build = [] 109 configuration.pkgs_to_build = []
110 configuration.pkgs_to_build.extend(args[1:]) 110 configuration.pkgs_to_build.extend(args[1:])
111 111
112 bb.cooker.BBCooker().cook(configuration) 112 cooker = bb.cooker.BBCooker(configuration)
113 cooker.cook()
114
113 115
114if __name__ == "__main__": 116if __name__ == "__main__":
115 main() 117 main()
116 sys.exit(0)
117 import profile
118 profile.run('main()', "profile.log")
119 import pstats
120 p = pstats.Stats('profile.log')
121 p.sort_stats('time')
122 p.print_stats()
123 p.print_callers()
diff --git a/bitbake/classes/base.bbclass b/bitbake/classes/base.bbclass
deleted file mode 100644
index cfb82a41cb..0000000000
--- a/bitbake/classes/base.bbclass
+++ /dev/null
@@ -1,79 +0,0 @@
1# Copyright (C) 2003 Chris Larson
2#
3# Permission is hereby granted, free of charge, to any person obtaining a
4# copy of this software and associated documentation files (the "Software"),
5# to deal in the Software without restriction, including without limitation
6# the rights to use, copy, modify, merge, publish, distribute, sublicense,
7# and/or sell copies of the Software, and to permit persons to whom the
8# Software is furnished to do so, subject to the following conditions:
9#
10# The above copyright notice and this permission notice shall be included
11# in all copies or substantial portions of the Software.
12#
13# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
16# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
17# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
18# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
19# OTHER DEALINGS IN THE SOFTWARE.
20
21die() {
22 bbfatal "$*"
23}
24
25bbnote() {
26 echo "NOTE:" "$*"
27}
28
29bbwarn() {
30 echo "WARNING:" "$*"
31}
32
33bbfatal() {
34 echo "FATAL:" "$*"
35 exit 1
36}
37
38bbdebug() {
39 test $# -ge 2 || {
40 echo "Usage: bbdebug level \"message\""
41 exit 1
42 }
43
44 test ${@bb.msg.debug_level} -ge $1 && {
45 shift
46 echo "DEBUG:" $*
47 }
48}
49
50addtask showdata
51do_showdata[nostamp] = "1"
52python do_showdata() {
53 import sys
54 # emit variables and shell functions
55 bb.data.emit_env(sys.__stdout__, d, True)
56 # emit the metadata which isnt valid shell
57 for e in bb.data.keys(d):
58 if bb.data.getVarFlag(e, 'python', d):
59 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, bb.data.getVar(e, d, 1)))
60}
61
62addtask listtasks
63do_listtasks[nostamp] = "1"
64python do_listtasks() {
65 import sys
66 for e in bb.data.keys(d):
67 if bb.data.getVarFlag(e, 'task', d):
68 sys.__stdout__.write("%s\n" % e)
69}
70
71addtask build
72do_build[dirs] = "${TOPDIR}"
73do_build[nostamp] = "1"
74python base_do_build () {
75 bb.note("The included, default BB base.bbclass does not define a useful default task.")
76 bb.note("Try running the 'listtasks' task against a .bb to see what tasks are defined.")
77}
78
79EXPORT_FUNCTIONS do_clean do_mrproper do_build
diff --git a/bitbake/conf/bitbake.conf b/bitbake/conf/bitbake.conf
deleted file mode 100644
index 19a3fe8ef8..0000000000
--- a/bitbake/conf/bitbake.conf
+++ /dev/null
@@ -1,58 +0,0 @@
1# Copyright (C) 2003 Chris Larson
2#
3# Permission is hereby granted, free of charge, to any person obtaining a
4# copy of this software and associated documentation files (the "Software"),
5# to deal in the Software without restriction, including without limitation
6# the rights to use, copy, modify, merge, publish, distribute, sublicense,
7# and/or sell copies of the Software, and to permit persons to whom the
8# Software is furnished to do so, subject to the following conditions:
9#
10# The above copyright notice and this permission notice shall be included
11# in all copies or substantial portions of the Software.
12#
13# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
16# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
17# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
18# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
19# OTHER DEALINGS IN THE SOFTWARE.
20
21B = "${S}"
22CVSDIR = "${DL_DIR}/cvs"
23DEPENDS = ""
24DEPLOY_DIR = "${TMPDIR}/deploy"
25DEPLOY_DIR_IMAGE = "${DEPLOY_DIR}/images"
26DL_DIR = "${TMPDIR}/downloads"
27FETCHCOMMAND = ""
28FETCHCOMMAND_cvs = "/usr/bin/env cvs -d${CVSROOT} co ${CVSCOOPTS} ${CVSMODULE}"
29FETCHCOMMAND_svn = "/usr/bin/env svn co ${SVNCOOPTS} ${SVNROOT} ${SVNMODULE}"
30FETCHCOMMAND_wget = "/usr/bin/env wget -t 5 --passive-ftp -P ${DL_DIR} ${URI}"
31FILESDIR = "${@bb.which(bb.data.getVar('FILESPATH', d, 1), '.')}"
32FILESPATH = "${FILE_DIRNAME}/${PF}:${FILE_DIRNAME}/${P}:${FILE_DIRNAME}/${PN}:${FILE_DIRNAME}/files:${FILE_DIRNAME}"
33FILE_DIRNAME = "${@os.path.dirname(bb.data.getVar('FILE', d))}"
34GITDIR = "${DL_DIR}/git"
35IMAGE_CMD = "_NO_DEFINED_IMAGE_TYPES_"
36IMAGE_ROOTFS = "${TMPDIR}/rootfs"
37MKTEMPCMD = "mktemp -q ${TMPBASE}"
38MKTEMPDIRCMD = "mktemp -d -q ${TMPBASE}"
39OVERRIDES = "local:${MACHINE}:${TARGET_OS}:${TARGET_ARCH}"
40P = "${PN}-${PV}"
41PF = "${PN}-${PV}-${PR}"
42PN = "${@bb.parse.BBHandler.vars_from_file(bb.data.getVar('FILE',d),d)[0] or 'defaultpkgname'}"
43PR = "${@bb.parse.BBHandler.vars_from_file(bb.data.getVar('FILE',d),d)[2] or 'r0'}"
44PROVIDES = ""
45PV = "${@bb.parse.BBHandler.vars_from_file(bb.data.getVar('FILE',d),d)[1] or '1.0'}"
46RESUMECOMMAND = ""
47RESUMECOMMAND_wget = "/usr/bin/env wget -c -t 5 --passive-ftp -P ${DL_DIR} ${URI}"
48S = "${WORKDIR}/${P}"
49SRC_URI = "file://${FILE}"
50STAMP = "${TMPDIR}/stamps/${PF}"
51SVNDIR = "${DL_DIR}/svn"
52T = "${WORKDIR}/temp"
53TARGET_ARCH = "${BUILD_ARCH}"
54TMPDIR = "${TOPDIR}/tmp"
55UPDATECOMMAND = ""
56UPDATECOMMAND_cvs = "/usr/bin/env cvs -d${CVSROOT} update ${CVSCOOPTS}"
57UPDATECOMMAND_svn = "/usr/bin/env svn update ${SVNCOOPTS}"
58WORKDIR = "${TMPDIR}/work/${PF}"
diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py
index a11af84b12..e34122a61e 100644
--- a/bitbake/lib/bb/__init__.py
+++ b/bitbake/lib/bb/__init__.py
@@ -21,7 +21,7 @@
21# with this program; if not, write to the Free Software Foundation, Inc., 21# with this program; if not, write to the Free Software Foundation, Inc.,
22# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 22# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
23 23
24__version__ = "1.7.4" 24__version__ = "1.8.1"
25 25
26__all__ = [ 26__all__ = [
27 27
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index bf6b612f32..3494c0a28e 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -188,7 +188,8 @@ def exec_func_shell(func, d):
188 maybe_fakeroot = "PATH=\"%s\" fakeroot " % bb.data.getVar("PATH", d, 1) 188 maybe_fakeroot = "PATH=\"%s\" fakeroot " % bb.data.getVar("PATH", d, 1)
189 else: 189 else:
190 maybe_fakeroot = '' 190 maybe_fakeroot = ''
191 ret = os.system('%ssh -e %s' % (maybe_fakeroot, runfile)) 191 lang_environment = "LC_ALL=C "
192 ret = os.system('%s%ssh -e %s' % (lang_environment, maybe_fakeroot, runfile))
192 try: 193 try:
193 os.chdir(prevdir) 194 os.chdir(prevdir)
194 except: 195 except:
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index 934d230e6a..335b221979 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -39,7 +39,7 @@ except ImportError:
39 import pickle 39 import pickle
40 bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.") 40 bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
41 41
42__cache_version__ = "125" 42__cache_version__ = "126"
43 43
44class Cache: 44class Cache:
45 """ 45 """
@@ -75,6 +75,9 @@ class Cache:
75 raise ValueError, 'Cache Version Mismatch' 75 raise ValueError, 'Cache Version Mismatch'
76 if version_data['BITBAKE_VER'] != bb.__version__: 76 if version_data['BITBAKE_VER'] != bb.__version__:
77 raise ValueError, 'Bitbake Version Mismatch' 77 raise ValueError, 'Bitbake Version Mismatch'
78 except EOFError:
79 bb.msg.note(1, bb.msg.domain.Cache, "Truncated cache found, rebuilding...")
80 self.depends_cache = {}
78 except (ValueError, KeyError): 81 except (ValueError, KeyError):
79 bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...") 82 bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...")
80 self.depends_cache = {} 83 self.depends_cache = {}
@@ -251,6 +254,7 @@ class Cache:
251 """ 254 """
252 255
253 pn = self.getVar('PN', file_name, True) 256 pn = self.getVar('PN', file_name, True)
257 pe = self.getVar('PE', file_name, True) or "0"
254 pv = self.getVar('PV', file_name, True) 258 pv = self.getVar('PV', file_name, True)
255 pr = self.getVar('PR', file_name, True) 259 pr = self.getVar('PR', file_name, True)
256 dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0") 260 dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
@@ -272,7 +276,7 @@ class Cache:
272 276
273 # build FileName to PackageName lookup table 277 # build FileName to PackageName lookup table
274 cacheData.pkg_fn[file_name] = pn 278 cacheData.pkg_fn[file_name] = pn
275 cacheData.pkg_pvpr[file_name] = (pv,pr) 279 cacheData.pkg_pepvpr[file_name] = (pe,pv,pr)
276 cacheData.pkg_dp[file_name] = dp 280 cacheData.pkg_dp[file_name] = dp
277 281
278 # Build forward and reverse provider hashes 282 # Build forward and reverse provider hashes
@@ -407,7 +411,7 @@ class CacheData:
407 self.possible_world = [] 411 self.possible_world = []
408 self.pkg_pn = {} 412 self.pkg_pn = {}
409 self.pkg_fn = {} 413 self.pkg_fn = {}
410 self.pkg_pvpr = {} 414 self.pkg_pepvpr = {}
411 self.pkg_dp = {} 415 self.pkg_dp = {}
412 self.pn_provides = {} 416 self.pn_provides = {}
413 self.all_depends = Set() 417 self.all_depends = Set()
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 8a9c588633..4b2a906133 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -31,29 +31,6 @@ import itertools
31parsespin = itertools.cycle( r'|/-\\' ) 31parsespin = itertools.cycle( r'|/-\\' )
32 32
33#============================================================================# 33#============================================================================#
34# BBStatistics
35#============================================================================#
36class BBStatistics:
37 """
38 Manage build statistics for one run
39 """
40 def __init__(self ):
41 self.attempt = 0
42 self.success = 0
43 self.fail = 0
44 self.deps = 0
45
46 def show( self ):
47 print "Build statistics:"
48 print " Attempted builds: %d" % self.attempt
49 if self.fail:
50 print " Failed builds: %d" % self.fail
51 if self.deps:
52 print " Dependencies not satisfied: %d" % self.deps
53 if self.fail or self.deps: return 1
54 else: return 0
55
56#============================================================================#
57# BBCooker 34# BBCooker
58#============================================================================# 35#============================================================================#
59class BBCooker: 36class BBCooker:
@@ -61,43 +38,61 @@ class BBCooker:
61 Manages one bitbake build run 38 Manages one bitbake build run
62 """ 39 """
63 40
64 Statistics = BBStatistics # make it visible from the shell 41 def __init__(self, configuration):
65
66 def __init__( self ):
67 self.build_cache_fail = []
68 self.build_cache = []
69 self.stats = BBStatistics()
70 self.status = None 42 self.status = None
71 43
72 self.cache = None 44 self.cache = None
73 self.bb_cache = None 45 self.bb_cache = None
74 46
47 self.configuration = configuration
48
49 if self.configuration.verbose:
50 bb.msg.set_verbose(True)
51
52 if self.configuration.debug:
53 bb.msg.set_debug_level(self.configuration.debug)
54 else:
55 bb.msg.set_debug_level(0)
56
57 if self.configuration.debug_domains:
58 bb.msg.set_debug_domains(self.configuration.debug_domains)
59
60 self.configuration.data = bb.data.init()
61
62 for f in self.configuration.file:
63 self.parseConfigurationFile( f )
64
65 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
66
67 if not self.configuration.cmd:
68 self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build"
69
70 #
71 # Special updated configuration we use for firing events
72 #
73 self.configuration.event_data = bb.data.createCopy(self.configuration.data)
74 bb.data.update_data(self.configuration.event_data)
75
75 def tryBuildPackage(self, fn, item, task, the_data, build_depends): 76 def tryBuildPackage(self, fn, item, task, the_data, build_depends):
76 """ 77 """
77 Build one task of a package, optionally build following task depends 78 Build one task of a package, optionally build following task depends
78 """ 79 """
79 bb.event.fire(bb.event.PkgStarted(item, the_data)) 80 bb.event.fire(bb.event.PkgStarted(item, the_data))
80 try: 81 try:
81 self.stats.attempt += 1
82 if not build_depends: 82 if not build_depends:
83 bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data) 83 bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data)
84 if not self.configuration.dry_run: 84 if not self.configuration.dry_run:
85 bb.build.exec_task('do_%s' % task, the_data) 85 bb.build.exec_task('do_%s' % task, the_data)
86 bb.event.fire(bb.event.PkgSucceeded(item, the_data)) 86 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
87 self.build_cache.append(fn)
88 return True 87 return True
89 except bb.build.FuncFailed: 88 except bb.build.FuncFailed:
90 self.stats.fail += 1
91 bb.msg.error(bb.msg.domain.Build, "task stack execution failed") 89 bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
92 bb.event.fire(bb.event.PkgFailed(item, the_data)) 90 bb.event.fire(bb.event.PkgFailed(item, the_data))
93 self.build_cache_fail.append(fn)
94 raise 91 raise
95 except bb.build.EventException, e: 92 except bb.build.EventException, e:
96 self.stats.fail += 1
97 event = e.args[1] 93 event = e.args[1]
98 bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event)) 94 bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
99 bb.event.fire(bb.event.PkgFailed(item, the_data)) 95 bb.event.fire(bb.event.PkgFailed(item, the_data))
100 self.build_cache_fail.append(fn)
101 raise 96 raise
102 97
103 def tryBuild( self, fn, build_depends): 98 def tryBuild( self, fn, build_depends):
@@ -112,12 +107,11 @@ class BBCooker:
112 item = self.status.pkg_fn[fn] 107 item = self.status.pkg_fn[fn]
113 108
114 if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data): 109 if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
115 self.build_cache.append(fn)
116 return True 110 return True
117 111
118 return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends) 112 return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends)
119 113
120 def showVersions( self ): 114 def showVersions(self):
121 pkg_pn = self.status.pkg_pn 115 pkg_pn = self.status.pkg_pn
122 preferred_versions = {} 116 preferred_versions = {}
123 latest_versions = {} 117 latest_versions = {}
@@ -136,11 +130,11 @@ class BBCooker:
136 latest = latest_versions[p] 130 latest = latest_versions[p]
137 131
138 if pref != latest: 132 if pref != latest:
139 prefstr = pref[0][0] + "-" + pref[0][1] 133 prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
140 else: 134 else:
141 prefstr = "" 135 prefstr = ""
142 136
143 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1], 137 print "%-30s %20s %20s" % (p, latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2],
144 prefstr) 138 prefstr)
145 139
146 140
@@ -192,8 +186,8 @@ class BBCooker:
192 taskdata.add_unresolved(localdata, self.status) 186 taskdata.add_unresolved(localdata, self.status)
193 except bb.providers.NoProvider: 187 except bb.providers.NoProvider:
194 sys.exit(1) 188 sys.exit(1)
195 rq = bb.runqueue.RunQueue() 189 rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
196 rq.prepare_runqueue(self, self.configuration.data, self.status, taskdata, runlist) 190 rq.prepare_runqueue()
197 191
198 seen_fnids = [] 192 seen_fnids = []
199 depends_file = file('depends.dot', 'w' ) 193 depends_file = file('depends.dot', 'w' )
@@ -371,98 +365,138 @@ class BBCooker:
371 except ValueError: 365 except ValueError:
372 bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority)) 366 bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
373 367
368 def buildSetVars(self):
369 """
370 Setup any variables needed before starting a build
371 """
372 if not bb.data.getVar("BUILDNAME", self.configuration.data):
373 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
374 bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
374 375
375 def cook(self, configuration): 376 def buildFile(self, buildfile):
376 """ 377 """
377 We are building stuff here. We do the building 378 Build the file matching regexp buildfile
378 from here. By default we try to execute task
379 build.
380 """ 379 """
381 380
382 self.configuration = configuration 381 bf = os.path.abspath(buildfile)
382 try:
383 os.stat(bf)
384 except OSError:
385 (filelist, masked) = self.collect_bbfiles()
386 regexp = re.compile(buildfile)
387 matches = []
388 for f in filelist:
389 if regexp.search(f) and os.path.isfile(f):
390 bf = f
391 matches.append(f)
392 if len(matches) != 1:
393 bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
394 for f in matches:
395 bb.msg.error(bb.msg.domain.Parsing, " %s" % f)
396 sys.exit(1)
397 bf = matches[0]
383 398
384 if self.configuration.verbose: 399 bbfile_data = bb.parse.handle(bf, self.configuration.data)
385 bb.msg.set_verbose(True)
386 400
387 if self.configuration.debug: 401 # Remove stamp for target if force mode active
388 bb.msg.set_debug_level(self.configuration.debug) 402 if self.configuration.force:
403 bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, bf))
404 bb.build.del_stamp('do_%s' % self.configuration.cmd, bbfile_data)
405
406 item = bb.data.getVar('PN', bbfile_data, 1)
407 try:
408 self.tryBuildPackage(bf, item, self.configuration.cmd, bbfile_data, True)
409 except bb.build.EventException:
410 bb.msg.error(bb.msg.domain.Build, "Build of '%s' failed" % item )
411
412 sys.exit(0)
413
414 def buildTargets(self, targets):
415 """
416 Attempt to build the targets specified
417 """
418
419 buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
420 bb.event.fire(bb.event.BuildStarted(buildname, targets, self.configuration.event_data))
421
422 localdata = data.createCopy(self.configuration.data)
423 bb.data.update_data(localdata)
424 bb.data.expandKeys(localdata)
425
426 taskdata = bb.taskdata.TaskData(self.configuration.abort)
427
428 runlist = []
429 try:
430 for k in targets:
431 taskdata.add_provider(localdata, self.status, k)
432 runlist.append([k, "do_%s" % self.configuration.cmd])
433 taskdata.add_unresolved(localdata, self.status)
434 except bb.providers.NoProvider:
435 sys.exit(1)
436
437 rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
438 rq.prepare_runqueue()
439 try:
440 failures = rq.execute_runqueue()
441 except runqueue.TaskFailure, fnids:
442 for fnid in fnids:
443 bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
444 sys.exit(1)
445 bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
446
447 sys.exit(0)
448
449 def updateCache(self):
450 # Import Psyco if available and not disabled
451 if not self.configuration.disable_psyco:
452 try:
453 import psyco
454 except ImportError:
455 bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
456 else:
457 psyco.bind( self.parse_bbfiles )
389 else: 458 else:
390 bb.msg.set_debug_level(0) 459 bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
391 460
392 if self.configuration.debug_domains: 461 self.status = bb.cache.CacheData()
393 bb.msg.set_debug_domains(self.configuration.debug_domains)
394 462
395 self.configuration.data = bb.data.init() 463 ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
464 self.status.ignored_dependencies = Set( ignore.split() )
396 465
397 for f in self.configuration.file: 466 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
398 self.parseConfigurationFile( f )
399 467
400 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) ) 468 bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
469 (filelist, masked) = self.collect_bbfiles()
470 self.parse_bbfiles(filelist, masked, self.myProgressCallback)
471 bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
401 472
402 if not self.configuration.cmd: 473 self.buildDepgraph()
403 self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build"
404 474
405 # 475 def cook(self):
406 # Special updated configuration we use for firing events 476 """
407 # 477 We are building stuff here. We do the building
408 self.configuration.event_data = bb.data.createCopy(self.configuration.data) 478 from here. By default we try to execute task
409 bb.data.update_data(self.configuration.event_data) 479 build.
480 """
410 481
411 if self.configuration.show_environment: 482 if self.configuration.show_environment:
412 self.showEnvironment() 483 self.showEnvironment()
413 sys.exit( 0 ) 484 sys.exit( 0 )
414 485
415 # inject custom variables 486 self.buildSetVars()
416 if not bb.data.getVar("BUILDNAME", self.configuration.data):
417 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
418 bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
419
420 buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
421 487
422 if self.configuration.interactive: 488 if self.configuration.interactive:
423 self.interactiveMode() 489 self.interactiveMode()
424 490
425 if self.configuration.buildfile is not None: 491 if self.configuration.buildfile is not None:
426 bf = os.path.abspath( self.configuration.buildfile ) 492 return self.buildFile(self.configuration.buildfile)
427 try:
428 os.stat(bf)
429 except OSError:
430 (filelist, masked) = self.collect_bbfiles()
431 regexp = re.compile(self.configuration.buildfile)
432 matches = []
433 for f in filelist:
434 if regexp.search(f) and os.path.isfile(f):
435 bf = f
436 matches.append(f)
437 if len(matches) != 1:
438 bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (self.configuration.buildfile, len(matches)))
439 for f in matches:
440 bb.msg.error(bb.msg.domain.Parsing, " %s" % f)
441 sys.exit(1)
442 bf = matches[0]
443
444 bbfile_data = bb.parse.handle(bf, self.configuration.data)
445
446 # Remove stamp for target if force mode active
447 if self.configuration.force:
448 bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, bf))
449 bb.build.del_stamp('do_%s' % self.configuration.cmd, bbfile_data)
450
451 item = bb.data.getVar('PN', bbfile_data, 1)
452 try:
453 self.tryBuildPackage(bf, item, self.configuration.cmd, bbfile_data, True)
454 except bb.build.EventException:
455 bb.msg.error(bb.msg.domain.Build, "Build of '%s' failed" % item )
456
457 sys.exit( self.stats.show() )
458 493
459 # initialise the parsing status now we know we will need deps 494 # initialise the parsing status now we know we will need deps
460 self.status = bb.cache.CacheData() 495 self.updateCache()
461 496
462 ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or "" 497 if self.configuration.parse_only:
463 self.status.ignored_dependencies = Set( ignore.split() ) 498 bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only. Exiting.")
464 499 return 0
465 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
466 500
467 pkgs_to_build = self.configuration.pkgs_to_build 501 pkgs_to_build = self.configuration.pkgs_to_build
468 502
@@ -475,30 +509,7 @@ class BBCooker:
475 print "for usage information." 509 print "for usage information."
476 sys.exit(0) 510 sys.exit(0)
477 511
478 # Import Psyco if available and not disabled
479 if not self.configuration.disable_psyco:
480 try:
481 import psyco
482 except ImportError:
483 bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
484 else:
485 psyco.bind( self.parse_bbfiles )
486 else:
487 bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
488
489 try: 512 try:
490 bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
491 (filelist, masked) = self.collect_bbfiles()
492 self.parse_bbfiles(filelist, masked, self.myProgressCallback)
493 bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
494 print
495 if self.configuration.parse_only:
496 bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only. Exiting.")
497 return
498
499
500 self.buildDepgraph()
501
502 if self.configuration.show_versions: 513 if self.configuration.show_versions:
503 self.showVersions() 514 self.showVersions()
504 sys.exit( 0 ) 515 sys.exit( 0 )
@@ -512,34 +523,7 @@ class BBCooker:
512 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps ) 523 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
513 sys.exit( 0 ) 524 sys.exit( 0 )
514 525
515 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.event_data)) 526 return self.buildTargets(pkgs_to_build)
516
517 localdata = data.createCopy(self.configuration.data)
518 bb.data.update_data(localdata)
519 bb.data.expandKeys(localdata)
520
521 taskdata = bb.taskdata.TaskData(self.configuration.abort)
522
523 runlist = []
524 try:
525 for k in pkgs_to_build:
526 taskdata.add_provider(localdata, self.status, k)
527 runlist.append([k, "do_%s" % self.configuration.cmd])
528 taskdata.add_unresolved(localdata, self.status)
529 except bb.providers.NoProvider:
530 sys.exit(1)
531
532 rq = bb.runqueue.RunQueue()
533 rq.prepare_runqueue(self, self.configuration.data, self.status, taskdata, runlist)
534 try:
535 failures = rq.execute_runqueue(self, self.configuration.data, self.status, taskdata, runlist)
536 except runqueue.TaskFailure, fnids:
537 for fnid in fnids:
538 bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
539 sys.exit(1)
540 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.event_data, failures))
541
542 sys.exit( self.stats.show() )
543 527
544 except KeyboardInterrupt: 528 except KeyboardInterrupt:
545 bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.") 529 bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
@@ -556,13 +540,17 @@ class BBCooker:
556 return bbfiles 540 return bbfiles
557 541
558 def find_bbfiles( self, path ): 542 def find_bbfiles( self, path ):
559 """Find all the .bb files in a directory (uses find)""" 543 """Find all the .bb files in a directory"""
560 findcmd = 'find ' + path + ' -name *.bb | grep -v SCCS/' 544 from os.path import join
561 try: 545
562 finddata = os.popen(findcmd) 546 found = []
563 except OSError: 547 for dir, dirs, files in os.walk(path):
564 return [] 548 for ignored in ('SCCS', 'CVS', '.svn'):
565 return finddata.readlines() 549 if ignored in dirs:
550 dirs.remove(ignored)
551 found += [join(dir,f) for f in files if f.endswith('.bb')]
552
553 return found
566 554
567 def collect_bbfiles( self ): 555 def collect_bbfiles( self ):
568 """Collect all available .bb build files""" 556 """Collect all available .bb build files"""
diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
index f1da98fd45..cfbda3e9fc 100644
--- a/bitbake/lib/bb/event.py
+++ b/bitbake/lib/bb/event.py
@@ -23,14 +23,13 @@ BitBake build tools.
23# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 23# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 24
25import os, re 25import os, re
26import bb.data
27import bb.utils 26import bb.utils
28 27
29class Event: 28class Event:
30 """Base class for events""" 29 """Base class for events"""
31 type = "Event" 30 type = "Event"
32 31
33 def __init__(self, d = bb.data.init()): 32 def __init__(self, d):
34 self._data = d 33 self._data = d
35 34
36 def getData(self): 35 def getData(self):
@@ -129,7 +128,7 @@ def getName(e):
129class PkgBase(Event): 128class PkgBase(Event):
130 """Base class for package events""" 129 """Base class for package events"""
131 130
132 def __init__(self, t, d = bb.data.init()): 131 def __init__(self, t, d):
133 self._pkg = t 132 self._pkg = t
134 Event.__init__(self, d) 133 Event.__init__(self, d)
135 134
diff --git a/bitbake/lib/bb/fetch/svn.py b/bitbake/lib/bb/fetch/svn.py
index 21be1412a6..120f4f8539 100644
--- a/bitbake/lib/bb/fetch/svn.py
+++ b/bitbake/lib/bb/fetch/svn.py
@@ -91,6 +91,12 @@ class Svn(Fetch):
91 elif ud.date != "now": 91 elif ud.date != "now":
92 options.append("-r {%s}" % ud.date) 92 options.append("-r {%s}" % ud.date)
93 93
94 if ud.user:
95 options.append("--username %s" % ud.user)
96
97 if ud.pswd:
98 options.append("--password %s" % ud.pswd)
99
94 localdata = data.createCopy(d) 100 localdata = data.createCopy(d)
95 data.setVar('OVERRIDES', "svn:%s" % data.getVar('OVERRIDES', localdata), localdata) 101 data.setVar('OVERRIDES', "svn:%s" % data.getVar('OVERRIDES', localdata), localdata)
96 data.update_data(localdata) 102 data.update_data(localdata)
diff --git a/bitbake/lib/bb/msg.py b/bitbake/lib/bb/msg.py
index bd7729731a..71b0b05b77 100644
--- a/bitbake/lib/bb/msg.py
+++ b/bitbake/lib/bb/msg.py
@@ -23,7 +23,7 @@ Message handling infrastructure for bitbake
23# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 23# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 24
25import sys, os, re, bb 25import sys, os, re, bb
26from bb import utils 26from bb import utils, event
27 27
28debug_level = {} 28debug_level = {}
29 29
@@ -42,6 +42,29 @@ domain = bb.utils.Enum(
42 'TaskData', 42 'TaskData',
43 'Util') 43 'Util')
44 44
45
46class MsgBase(bb.event.Event):
47 """Base class for messages"""
48
49 def __init__(self, msg, d ):
50 self._message = msg
51 event.Event.__init__(self, d)
52
53class MsgDebug(MsgBase):
54 """Debug Message"""
55
56class MsgNote(MsgBase):
57 """Note Message"""
58
59class MsgWarn(MsgBase):
60 """Warning Message"""
61
62class MsgError(MsgBase):
63 """Error Message"""
64
65class MsgFatal(MsgBase):
66 """Fatal Message"""
67
45# 68#
46# Message control functions 69# Message control functions
47# 70#
@@ -71,6 +94,7 @@ def set_debug_domains(domains):
71 94
72def debug(level, domain, msg, fn = None): 95def debug(level, domain, msg, fn = None):
73 if debug_level[domain] >= level: 96 if debug_level[domain] >= level:
97 bb.event.fire(MsgDebug(msg, None))
74 print 'DEBUG: ' + msg 98 print 'DEBUG: ' + msg
75 99
76def note(level, domain, msg, fn = None): 100def note(level, domain, msg, fn = None):
@@ -91,17 +115,22 @@ def fatal(domain, msg, fn = None):
91# 115#
92def std_debug(lvl, msg): 116def std_debug(lvl, msg):
93 if debug_level['default'] >= lvl: 117 if debug_level['default'] >= lvl:
118 bb.event.fire(MsgDebug(msg, None))
94 print 'DEBUG: ' + msg 119 print 'DEBUG: ' + msg
95 120
96def std_note(msg): 121def std_note(msg):
122 bb.event.fire(MsgNote(msg, None))
97 print 'NOTE: ' + msg 123 print 'NOTE: ' + msg
98 124
99def std_warn(msg): 125def std_warn(msg):
126 bb.event.fire(MsgWarn(msg, None))
100 print 'WARNING: ' + msg 127 print 'WARNING: ' + msg
101 128
102def std_error(msg): 129def std_error(msg):
130 bb.event.fire(MsgError(msg, None))
103 print 'ERROR: ' + msg 131 print 'ERROR: ' + msg
104 132
105def std_fatal(msg): 133def std_fatal(msg):
134 bb.event.fire(MsgFatal(msg, None))
106 print 'ERROR: ' + msg 135 print 'ERROR: ' + msg
107 sys.exit(1) 136 sys.exit(1)
diff --git a/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
index 1ae673079d..0e05928d84 100644
--- a/bitbake/lib/bb/parse/parse_py/ConfHandler.py
+++ b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
@@ -161,6 +161,12 @@ def handle(fn, data, include = 0):
161 return data 161 return data
162 162
163def feeder(lineno, s, fn, data): 163def feeder(lineno, s, fn, data):
164 def getFunc(groupd, key, data):
165 if 'flag' in groupd and groupd['flag'] != None:
166 return bb.data.getVarFlag(key, groupd['flag'], data)
167 else:
168 return bb.data.getVar(key, data)
169
164 m = __config_regexp__.match(s) 170 m = __config_regexp__.match(s)
165 if m: 171 if m:
166 groupd = m.groupdict() 172 groupd = m.groupdict()
@@ -168,19 +174,19 @@ def feeder(lineno, s, fn, data):
168 if "exp" in groupd and groupd["exp"] != None: 174 if "exp" in groupd and groupd["exp"] != None:
169 bb.data.setVarFlag(key, "export", 1, data) 175 bb.data.setVarFlag(key, "export", 1, data)
170 if "ques" in groupd and groupd["ques"] != None: 176 if "ques" in groupd and groupd["ques"] != None:
171 val = bb.data.getVar(key, data) 177 val = getFunc(groupd, key, data)
172 if val == None: 178 if val == None:
173 val = groupd["value"] 179 val = groupd["value"]
174 elif "colon" in groupd and groupd["colon"] != None: 180 elif "colon" in groupd and groupd["colon"] != None:
175 val = bb.data.expand(groupd["value"], data) 181 val = bb.data.expand(groupd["value"], data)
176 elif "append" in groupd and groupd["append"] != None: 182 elif "append" in groupd and groupd["append"] != None:
177 val = "%s %s" % ((bb.data.getVar(key, data) or ""), groupd["value"]) 183 val = "%s %s" % ((getFunc(groupd, key, data) or ""), groupd["value"])
178 elif "prepend" in groupd and groupd["prepend"] != None: 184 elif "prepend" in groupd and groupd["prepend"] != None:
179 val = "%s %s" % (groupd["value"], (bb.data.getVar(key, data) or "")) 185 val = "%s %s" % (groupd["value"], (getFunc(groupd, key, data) or ""))
180 elif "postdot" in groupd and groupd["postdot"] != None: 186 elif "postdot" in groupd and groupd["postdot"] != None:
181 val = "%s%s" % ((bb.data.getVar(key, data) or ""), groupd["value"]) 187 val = "%s%s" % ((getFunc(groupd, key, data) or ""), groupd["value"])
182 elif "predot" in groupd and groupd["predot"] != None: 188 elif "predot" in groupd and groupd["predot"] != None:
183 val = "%s%s" % (groupd["value"], (bb.data.getVar(key, data) or "")) 189 val = "%s%s" % (groupd["value"], (getFunc(groupd, key, data) or ""))
184 else: 190 else:
185 val = groupd["value"] 191 val = groupd["value"]
186 if 'flag' in groupd and groupd['flag'] != None: 192 if 'flag' in groupd and groupd['flag'] != None:
diff --git a/bitbake/lib/bb/providers.py b/bitbake/lib/bb/providers.py
index fdd6cd10d1..78f45122ff 100644
--- a/bitbake/lib/bb/providers.py
+++ b/bitbake/lib/bb/providers.py
@@ -61,19 +61,27 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
61 61
62 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, localdata, True) 62 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, localdata, True)
63 if preferred_v: 63 if preferred_v:
64 m = re.match('(.*)_(.*)', preferred_v) 64 m = re.match('(\d+:)*(.*)(_.*)*', preferred_v)
65 if m: 65 if m:
66 preferred_v = m.group(1) 66 if m.group(1):
67 preferred_r = m.group(2) 67 preferred_e = int(m.group(1)[:-1])
68 else:
69 preferred_e = None
70 preferred_v = m.group(2)
71 if m.group(3):
72 preferred_r = m.group(3)[1:]
73 else:
74 preferred_r = None
68 else: 75 else:
76 preferred_e = None
69 preferred_r = None 77 preferred_r = None
70 78
71 for file_set in tmp_pn: 79 for file_set in tmp_pn:
72 for f in file_set: 80 for f in file_set:
73 pv,pr = dataCache.pkg_pvpr[f] 81 pe,pv,pr = dataCache.pkg_pepvpr[f]
74 if preferred_v == pv and (preferred_r == pr or preferred_r == None): 82 if preferred_v == pv and (preferred_r == pr or preferred_r == None) and (preferred_e == pe or preferred_e == None):
75 preferred_file = f 83 preferred_file = f
76 preferred_ver = (pv, pr) 84 preferred_ver = (pe, pv, pr)
77 break 85 break
78 if preferred_file: 86 if preferred_file:
79 break; 87 break;
@@ -81,6 +89,8 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
81 pv_str = '%s-%s' % (preferred_v, preferred_r) 89 pv_str = '%s-%s' % (preferred_v, preferred_r)
82 else: 90 else:
83 pv_str = preferred_v 91 pv_str = preferred_v
92 if not (preferred_e is None):
93 pv_str = '%s:%s' % (preferred_e, pv_str)
84 itemstr = "" 94 itemstr = ""
85 if item: 95 if item:
86 itemstr = " (for item %s)" % item 96 itemstr = " (for item %s)" % item
@@ -97,11 +107,11 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
97 latest_p = 0 107 latest_p = 0
98 latest_f = None 108 latest_f = None
99 for file_name in files: 109 for file_name in files:
100 pv,pr = dataCache.pkg_pvpr[file_name] 110 pe,pv,pr = dataCache.pkg_pepvpr[file_name]
101 dp = dataCache.pkg_dp[file_name] 111 dp = dataCache.pkg_dp[file_name]
102 112
103 if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p): 113 if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pe, pv, pr)) < 0)) or (dp > latest_p):
104 latest = (pv, pr) 114 latest = (pe, pv, pr)
105 latest_f = file_name 115 latest_f = file_name
106 latest_p = dp 116 latest_p = dp
107 if preferred_file is None: 117 if preferred_file is None:
@@ -110,10 +120,7 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
110 120
111 return (latest,latest_f,preferred_ver, preferred_file) 121 return (latest,latest_f,preferred_ver, preferred_file)
112 122
113# 123def filterProviders(providers, item, cfgData, dataCache):
114# RP - build_cache_fail needs to move elsewhere
115#
116def filterProviders(providers, item, cfgData, dataCache, build_cache_fail = {}):
117 """ 124 """
118 Take a list of providers and filter/reorder according to the 125 Take a list of providers and filter/reorder according to the
119 environment variables and previous build results 126 environment variables and previous build results
@@ -135,12 +142,6 @@ def filterProviders(providers, item, cfgData, dataCache, build_cache_fail = {}):
135 preferred_versions[pn] = bb.providers.findBestProvider(pn, cfgData, dataCache, pkg_pn, item)[2:4] 142 preferred_versions[pn] = bb.providers.findBestProvider(pn, cfgData, dataCache, pkg_pn, item)[2:4]
136 eligible.append(preferred_versions[pn][1]) 143 eligible.append(preferred_versions[pn][1])
137 144
138
139 for p in eligible:
140 if p in build_cache_fail:
141 bb.msg.debug(1, bb.msg.domain.Provider, "rejecting already-failed %s" % p)
142 eligible.remove(p)
143
144 if len(eligible) == 0: 145 if len(eligible) == 0:
145 bb.msg.error(bb.msg.domain.Provider, "no eligible providers for %s" % item) 146 bb.msg.error(bb.msg.domain.Provider, "no eligible providers for %s" % item)
146 return 0 147 return 0
@@ -162,7 +163,7 @@ def filterProviders(providers, item, cfgData, dataCache, build_cache_fail = {}):
162 # if so, bump it to the head of the queue 163 # if so, bump it to the head of the queue
163 for p in providers: 164 for p in providers:
164 pn = dataCache.pkg_fn[p] 165 pn = dataCache.pkg_fn[p]
165 pv, pr = dataCache.pkg_pvpr[p] 166 pe, pv, pr = dataCache.pkg_pepvpr[p]
166 167
167 stamp = '%s.do_populate_staging' % dataCache.stamp[p] 168 stamp = '%s.do_populate_staging' % dataCache.stamp[p]
168 if os.path.exists(stamp): 169 if os.path.exists(stamp):
@@ -171,7 +172,11 @@ def filterProviders(providers, item, cfgData, dataCache, build_cache_fail = {}):
171 # package was made ineligible by already-failed check 172 # package was made ineligible by already-failed check
172 continue 173 continue
173 oldver = "%s-%s" % (pv, pr) 174 oldver = "%s-%s" % (pv, pr)
174 newver = '-'.join(newvers) 175 if pe > 0:
176 oldver = "%s:%s" % (pe, oldver)
177 newver = "%s-%s" % (newvers[1], newvers[2])
178 if newvers[0] > 0:
179 newver = "%s:%s" % (newvers[0], newver)
175 if (newver != oldver): 180 if (newver != oldver):
176 extra_chat = "%s (%s) already staged but upgrading to %s to satisfy %s" % (pn, oldver, newver, item) 181 extra_chat = "%s (%s) already staged but upgrading to %s to satisfy %s" % (pn, oldver, newver, item)
177 else: 182 else:
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index ec94b0f8ba..059f800b65 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -25,20 +25,47 @@ Handles preparation and execution of a queue of tasks
25from bb import msg, data, fetch, event, mkdirhier, utils 25from bb import msg, data, fetch, event, mkdirhier, utils
26from sets import Set 26from sets import Set
27import bb, os, sys 27import bb, os, sys
28import signal
28 29
29class TaskFailure(Exception): 30class TaskFailure(Exception):
30 """Exception raised when a task in a runqueue fails""" 31 """Exception raised when a task in a runqueue fails"""
31 def __init__(self, x): 32 def __init__(self, x):
32 self.args = x 33 self.args = x
33 34
35
36class RunQueueStats:
37 """
38 Holds statistics on the tasks handled by the associated runQueue
39 """
40 def __init__(self):
41 self.completed = 0
42 self.skipped = 0
43 self.failed = 0
44
45 def taskFailed(self):
46 self.failed = self.failed + 1
47
48 def taskCompleted(self):
49 self.completed = self.completed + 1
50
51 def taskSkipped(self):
52 self.skipped = self.skipped + 1
53
34class RunQueue: 54class RunQueue:
35 """ 55 """
36 BitBake Run Queue implementation 56 BitBake Run Queue implementation
37 """ 57 """
38 def __init__(self): 58 def __init__(self, cooker, cfgData, dataCache, taskData, targets):
39 self.reset_runqueue() 59 self.reset_runqueue()
60 self.cooker = cooker
61 self.dataCache = dataCache
62 self.taskData = taskData
63 self.targets = targets
64
65 self.number_tasks = int(bb.data.getVar("BB_NUMBER_THREADS", cfgData) or 1)
40 66
41 def reset_runqueue(self): 67 def reset_runqueue(self):
68
42 self.runq_fnid = [] 69 self.runq_fnid = []
43 self.runq_task = [] 70 self.runq_task = []
44 self.runq_depends = [] 71 self.runq_depends = []
@@ -46,16 +73,15 @@ class RunQueue:
46 self.runq_weight = [] 73 self.runq_weight = []
47 self.prio_map = [] 74 self.prio_map = []
48 75
49 def get_user_idstring(self, task, taskData): 76 def get_user_idstring(self, task):
50 fn = taskData.fn_index[self.runq_fnid[task]] 77 fn = self.taskData.fn_index[self.runq_fnid[task]]
51 taskname = self.runq_task[task] 78 taskname = self.runq_task[task]
52 return "%s, %s" % (fn, taskname) 79 return "%s, %s" % (fn, taskname)
53 80
54 def prepare_runqueue(self, cooker, cfgData, dataCache, taskData, targets): 81 def prepare_runqueue(self):
55 """ 82 """
56 Turn a set of taskData into a RunQueue and compute data needed 83 Turn a set of taskData into a RunQueue and compute data needed
57 to optimise the execution order. 84 to optimise the execution order.
58 targets is list of paired values - a provider name and the task to run
59 """ 85 """
60 86
61 depends = [] 87 depends = []
@@ -63,12 +89,14 @@ class RunQueue:
63 runq_build = [] 89 runq_build = []
64 runq_done = [] 90 runq_done = []
65 91
92 taskData = self.taskData
93
66 bb.msg.note(1, bb.msg.domain.RunQueue, "Preparing Runqueue") 94 bb.msg.note(1, bb.msg.domain.RunQueue, "Preparing Runqueue")
67 95
68 for task in range(len(taskData.tasks_name)): 96 for task in range(len(taskData.tasks_name)):
69 fnid = taskData.tasks_fnid[task] 97 fnid = taskData.tasks_fnid[task]
70 fn = taskData.fn_index[fnid] 98 fn = taskData.fn_index[fnid]
71 task_deps = dataCache.task_deps[fn] 99 task_deps = self.dataCache.task_deps[fn]
72 100
73 if fnid not in taskData.failed_fnids: 101 if fnid not in taskData.failed_fnids:
74 102
@@ -94,6 +122,15 @@ class RunQueue:
94 dep = taskData.fn_index[depdata] 122 dep = taskData.fn_index[depdata]
95 depends.append(taskData.gettask_id(dep, taskname)) 123 depends.append(taskData.gettask_id(dep, taskname))
96 124
125 idepends = taskData.tasks_idepends[task]
126 for idepend in idepends:
127 depid = int(idepend.split(":")[0])
128 if depid in taskData.build_targets:
129 depdata = taskData.build_targets[depid][0]
130 if depdata:
131 dep = taskData.fn_index[depdata]
132 depends.append(taskData.gettask_id(dep, idepend.split(":")[1]))
133
97 def add_recursive_build(depid): 134 def add_recursive_build(depid):
98 """ 135 """
99 Add build depends of depid to depends 136 Add build depends of depid to depends
@@ -197,7 +234,7 @@ class RunQueue:
197 for depend in depends: 234 for depend in depends:
198 mark_active(depend, depth+1) 235 mark_active(depend, depth+1)
199 236
200 for target in targets: 237 for target in self.targets:
201 targetid = taskData.getbuild_id(target[0]) 238 targetid = taskData.getbuild_id(target[0])
202 239
203 if targetid not in taskData.build_targets: 240 if targetid not in taskData.build_targets:
@@ -209,10 +246,10 @@ class RunQueue:
209 fnid = taskData.build_targets[targetid][0] 246 fnid = taskData.build_targets[targetid][0]
210 247
211 # Remove stamps for targets if force mode active 248 # Remove stamps for targets if force mode active
212 if cooker.configuration.force: 249 if self.cooker.configuration.force:
213 fn = taskData.fn_index[fnid] 250 fn = taskData.fn_index[fnid]
214 bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (target[1], fn)) 251 bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (target[1], fn))
215 bb.build.del_stamp(target[1], dataCache, fn) 252 bb.build.del_stamp(target[1], self.dataCache, fn)
216 253
217 if fnid in taskData.failed_fnids: 254 if fnid in taskData.failed_fnids:
218 continue 255 continue
@@ -299,18 +336,18 @@ class RunQueue:
299 seen.append(taskid) 336 seen.append(taskid)
300 for revdep in self.runq_revdeps[taskid]: 337 for revdep in self.runq_revdeps[taskid]:
301 if runq_done[revdep] == 0 and revdep not in seen and not finish: 338 if runq_done[revdep] == 0 and revdep not in seen and not finish:
302 bb.msg.error(bb.msg.domain.RunQueue, "Task %s (%s) (depends: %s)" % (revdep, self.get_user_idstring(revdep, taskData), self.runq_depends[revdep])) 339 bb.msg.error(bb.msg.domain.RunQueue, "Task %s (%s) (depends: %s)" % (revdep, self.get_user_idstring(revdep), self.runq_depends[revdep]))
303 if revdep in deps_seen: 340 if revdep in deps_seen:
304 bb.msg.error(bb.msg.domain.RunQueue, "Chain ends at Task %s (%s)" % (revdep, self.get_user_idstring(revdep, taskData))) 341 bb.msg.error(bb.msg.domain.RunQueue, "Chain ends at Task %s (%s)" % (revdep, self.get_user_idstring(revdep)))
305 finish = True 342 finish = True
306 return 343 return
307 for dep in self.runq_depends[revdep]: 344 for dep in self.runq_depends[revdep]:
308 deps_seen.append(dep) 345 deps_seen.append(dep)
309 print_chain(revdep, finish) 346 print_chain(revdep, finish)
310 print_chain(task, False) 347 print_chain(task, False)
311 bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) not processed!\nThis is probably a circular dependency (the chain might be printed above)." % (task, self.get_user_idstring(task, taskData))) 348 bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) not processed!\nThis is probably a circular dependency (the chain might be printed above)." % (task, self.get_user_idstring(task)))
312 if runq_weight1[task] != 0: 349 if runq_weight1[task] != 0:
313 bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) count not zero!" % (task, self.get_user_idstring(task, taskData))) 350 bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) count not zero!" % (task, self.get_user_idstring(task)))
314 351
315 # Make a weight sorted map 352 # Make a weight sorted map
316 from copy import deepcopy 353 from copy import deepcopy
@@ -328,7 +365,7 @@ class RunQueue:
328 365
329 #self.dump_data(taskData) 366 #self.dump_data(taskData)
330 367
331 def execute_runqueue(self, cooker, cfgData, dataCache, taskData, runlist): 368 def execute_runqueue(self):
332 """ 369 """
333 Run the tasks in a queue prepared by prepare_runqueue 370 Run the tasks in a queue prepared by prepare_runqueue
334 Upon failure, optionally try to recover the build using any alternate providers 371 Upon failure, optionally try to recover the build using any alternate providers
@@ -337,35 +374,86 @@ class RunQueue:
337 374
338 failures = 0 375 failures = 0
339 while 1: 376 while 1:
340 failed_fnids = self.execute_runqueue_internal(cooker, cfgData, dataCache, taskData) 377 failed_fnids = []
378 try:
379 self.execute_runqueue_internal()
380 finally:
381 if self.master_process:
382 failed_fnids = self.finish_runqueue()
341 if len(failed_fnids) == 0: 383 if len(failed_fnids) == 0:
342 return failures 384 return failures
343 if taskData.abort: 385 if self.taskData.abort:
344 raise bb.runqueue.TaskFailure(failed_fnids) 386 raise bb.runqueue.TaskFailure(failed_fnids)
345 for fnid in failed_fnids: 387 for fnid in failed_fnids:
346 #print "Failure: %s %s %s" % (fnid, taskData.fn_index[fnid], self.runq_task[fnid]) 388 #print "Failure: %s %s %s" % (fnid, self.taskData.fn_index[fnid], self.runq_task[fnid])
347 taskData.fail_fnid(fnid) 389 self.taskData.fail_fnid(fnid)
348 failures = failures + 1 390 failures = failures + 1
349 self.reset_runqueue() 391 self.reset_runqueue()
350 self.prepare_runqueue(cooker, cfgData, dataCache, taskData, runlist) 392 self.prepare_runqueue()
393
394 def execute_runqueue_initVars(self):
395
396 self.stats = RunQueueStats()
397
398 self.active_builds = 0
399 self.runq_buildable = []
400 self.runq_running = []
401 self.runq_complete = []
402 self.build_pids = {}
403 self.failed_fnids = []
404 self.master_process = True
351 405
352 def execute_runqueue_internal(self, cooker, cfgData, dataCache, taskData): 406 # Mark initial buildable tasks
407 for task in range(len(self.runq_fnid)):
408 self.runq_running.append(0)
409 self.runq_complete.append(0)
410 if len(self.runq_depends[task]) == 0:
411 self.runq_buildable.append(1)
412 else:
413 self.runq_buildable.append(0)
414
415 def task_complete(self, task):
416 """
417 Mark a task as completed
418 Look at the reverse dependencies and mark any task with
419 completed dependencies as buildable
420 """
421 self.runq_complete[task] = 1
422 for revdep in self.runq_revdeps[task]:
423 if self.runq_running[revdep] == 1:
424 continue
425 if self.runq_buildable[revdep] == 1:
426 continue
427 alldeps = 1
428 for dep in self.runq_depends[revdep]:
429 if self.runq_complete[dep] != 1:
430 alldeps = 0
431 if alldeps == 1:
432 self.runq_buildable[revdep] = 1
433 fn = self.taskData.fn_index[self.runq_fnid[revdep]]
434 taskname = self.runq_task[revdep]
435 bb.msg.debug(1, bb.msg.domain.RunQueue, "Marking task %s (%s, %s) as buildable" % (revdep, fn, taskname))
436
437 def get_next_task(self):
438 """
439 Return the id of the highest priority task that is buildable
440 """
441 for task1 in range(len(self.runq_fnid)):
442 task = self.prio_map[task1]
443 if self.runq_running[task] == 1:
444 continue
445 if self.runq_buildable[task] == 1:
446 return task
447 return None
448
449 def execute_runqueue_internal(self):
353 """ 450 """
354 Run the tasks in a queue prepared by prepare_runqueue 451 Run the tasks in a queue prepared by prepare_runqueue
355 """ 452 """
356 import signal
357 453
358 bb.msg.note(1, bb.msg.domain.RunQueue, "Executing runqueue") 454 bb.msg.note(1, bb.msg.domain.RunQueue, "Executing runqueue")
359 455
360 active_builds = 0 456 self.execute_runqueue_initVars()
361 tasks_completed = 0
362 tasks_skipped = 0
363
364 runq_buildable = []
365 runq_running = []
366 runq_complete = []
367 build_pids = {}
368 failed_fnids = []
369 457
370 if len(self.runq_fnid) == 0: 458 if len(self.runq_fnid) == 0:
371 # nothing to do 459 # nothing to do
@@ -374,144 +462,103 @@ class RunQueue:
374 def sigint_handler(signum, frame): 462 def sigint_handler(signum, frame):
375 raise KeyboardInterrupt 463 raise KeyboardInterrupt
376 464
377 def get_next_task(data): 465 while True:
378 """ 466 task = self.get_next_task()
379 Return the id of the highest priority task that is buildable 467 if task is not None:
380 """ 468 fn = self.taskData.fn_index[self.runq_fnid[task]]
381 for task1 in range(len(data.runq_fnid)): 469
382 task = data.prio_map[task1] 470 taskname = self.runq_task[task]
383 if runq_running[task] == 1: 471 if bb.build.stamp_is_current(taskname, self.dataCache, fn):
472 bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task)))
473 self.runq_running[task] = 1
474 self.task_complete(task)
475 self.stats.taskCompleted()
476 self.stats.taskSkipped()
384 continue 477 continue
385 if runq_buildable[task] == 1:
386 return task
387 return None
388 478
389 def task_complete(data, task): 479 bb.msg.note(1, bb.msg.domain.RunQueue, "Running task %d of %d (ID: %s, %s)" % (self.stats.completed + self.active_builds + 1, len(self.runq_fnid), task, self.get_user_idstring(task)))
390 """ 480 try:
391 Mark a task as completed 481 pid = os.fork()
392 Look at the reverse dependencies and mark any task with 482 except OSError, e:
393 completed dependencies as buildable 483 bb.msg.fatal(bb.msg.domain.RunQueue, "fork failed: %d (%s)" % (e.errno, e.strerror))
394 """ 484 if pid == 0:
395 runq_complete[task] = 1 485 # Bypass master process' handling
396 for revdep in data.runq_revdeps[task]: 486 self.master_process = False
397 if runq_running[revdep] == 1: 487 # Stop Ctrl+C being sent to children
398 continue 488 # signal.signal(signal.SIGINT, signal.SIG_IGN)
399 if runq_buildable[revdep] == 1: 489 # Make the child the process group leader
490 os.setpgid(0, 0)
491 sys.stdin = open('/dev/null', 'r')
492 self.cooker.configuration.cmd = taskname[3:]
493 try:
494 self.cooker.tryBuild(fn, False)
495 except bb.build.EventException:
496 bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed")
497 sys.exit(1)
498 except:
499 bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed")
500 raise
501 sys.exit(0)
502 self.build_pids[pid] = task
503 self.runq_running[task] = 1
504 self.active_builds = self.active_builds + 1
505 if self.active_builds < self.number_tasks:
400 continue 506 continue
401 alldeps = 1 507 if self.active_builds > 0:
402 for dep in data.runq_depends[revdep]: 508 result = os.waitpid(-1, 0)
403 if runq_complete[dep] != 1: 509 self.active_builds = self.active_builds - 1
404 alldeps = 0 510 task = self.build_pids[result[0]]
405 if alldeps == 1: 511 if result[1] != 0:
406 runq_buildable[revdep] = 1 512 del self.build_pids[result[0]]
407 fn = taskData.fn_index[self.runq_fnid[revdep]] 513 bb.msg.error(bb.msg.domain.RunQueue, "Task %s (%s) failed" % (task, self.get_user_idstring(task)))
408 taskname = self.runq_task[revdep] 514 self.failed_fnids.append(self.runq_fnid[task])
409 bb.msg.debug(1, bb.msg.domain.RunQueue, "Marking task %s (%s, %s) as buildable" % (revdep, fn, taskname)) 515 self.stats.taskFailed()
410 516 break
411 # Mark initial buildable tasks 517 self.task_complete(task)
412 for task in range(len(self.runq_fnid)): 518 self.stats.taskCompleted()
413 runq_running.append(0) 519 del self.build_pids[result[0]]
414 runq_complete.append(0) 520 continue
415 if len(self.runq_depends[task]) == 0: 521 return
416 runq_buildable.append(1)
417 else:
418 runq_buildable.append(0)
419
420
421 number_tasks = int(bb.data.getVar("BB_NUMBER_THREADS", cfgData) or 1)
422 522
523 def finish_runqueue(self):
423 try: 524 try:
424 while 1: 525 while self.active_builds > 0:
425 task = get_next_task(self) 526 bb.msg.note(1, bb.msg.domain.RunQueue, "Waiting for %s active tasks to finish" % self.active_builds)
426 if task is not None: 527 tasknum = 1
427 fn = taskData.fn_index[self.runq_fnid[task]] 528 for k, v in self.build_pids.iteritems():
428 taskname = self.runq_task[task] 529 bb.msg.note(1, bb.msg.domain.RunQueue, "%s: %s (%s)" % (tasknum, self.get_user_idstring(v), k))
429 530 tasknum = tasknum + 1
430 if bb.build.stamp_is_current(taskname, dataCache, fn): 531 result = os.waitpid(-1, 0)
431 bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task, taskData))) 532 task = self.build_pids[result[0]]
432 runq_running[task] = 1 533 if result[1] != 0:
433 task_complete(self, task) 534 bb.msg.error(bb.msg.domain.RunQueue, "Task %s (%s) failed" % (task, self.get_user_idstring(task)))
434 tasks_completed = tasks_completed + 1 535 self.failed_fnids.append(self.runq_fnid[task])
435 tasks_skipped = tasks_skipped + 1 536 self.stats.taskFailed()
436 continue 537 del self.build_pids[result[0]]
437 538 self.active_builds = self.active_builds - 1
438 bb.msg.note(1, bb.msg.domain.RunQueue, "Running task %d of %d (ID: %s, %s)" % (tasks_completed + active_builds + 1, len(self.runq_fnid), task, self.get_user_idstring(task, taskData))) 539 if len(self.failed_fnids) > 0:
439 try: 540 return self.failed_fnids
440 pid = os.fork() 541 except KeyboardInterrupt:
441 except OSError, e: 542 bb.msg.note(1, bb.msg.domain.RunQueue, "Sending SIGINT to remaining %s tasks" % self.active_builds)
442 bb.msg.fatal(bb.msg.domain.RunQueue, "fork failed: %d (%s)" % (e.errno, e.strerror)) 543 for k, v in self.build_pids.iteritems():
443 if pid == 0: 544 try:
444 # Bypass finally below
445 active_builds = 0
446 # Stop Ctrl+C being sent to children
447 # signal.signal(signal.SIGINT, signal.SIG_IGN)
448 # Make the child the process group leader
449 os.setpgid(0, 0)
450 sys.stdin = open('/dev/null', 'r')
451 cooker.configuration.cmd = taskname[3:]
452 try:
453 cooker.tryBuild(fn, False)
454 except bb.build.EventException:
455 bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed")
456 sys.exit(1)
457 except:
458 bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed")
459 raise
460 sys.exit(0)
461 build_pids[pid] = task
462 runq_running[task] = 1
463 active_builds = active_builds + 1
464 if active_builds < number_tasks:
465 continue
466 if active_builds > 0:
467 result = os.waitpid(-1, 0)
468 active_builds = active_builds - 1
469 task = build_pids[result[0]]
470 if result[1] != 0:
471 del build_pids[result[0]]
472 bb.msg.error(bb.msg.domain.RunQueue, "Task %s (%s) failed" % (task, self.get_user_idstring(task, taskData)))
473 failed_fnids.append(self.runq_fnid[task])
474 break
475 task_complete(self, task)
476 tasks_completed = tasks_completed + 1
477 del build_pids[result[0]]
478 continue
479 break
480 finally:
481 try:
482 while active_builds > 0:
483 bb.msg.note(1, bb.msg.domain.RunQueue, "Waiting for %s active tasks to finish" % active_builds)
484 tasknum = 1
485 for k, v in build_pids.iteritems():
486 bb.msg.note(1, bb.msg.domain.RunQueue, "%s: %s (%s)" % (tasknum, self.get_user_idstring(v, taskData), k))
487 tasknum = tasknum + 1
488 result = os.waitpid(-1, 0)
489 task = build_pids[result[0]]
490 if result[1] != 0:
491 bb.msg.error(bb.msg.domain.RunQueue, "Task %s (%s) failed" % (task, self.get_user_idstring(task, taskData)))
492 failed_fnids.append(self.runq_fnid[task])
493 del build_pids[result[0]]
494 active_builds = active_builds - 1
495 if len(failed_fnids) > 0:
496 return failed_fnids
497 except:
498 bb.msg.note(1, bb.msg.domain.RunQueue, "Sending SIGINT to remaining %s tasks" % active_builds)
499 for k, v in build_pids.iteritems():
500 os.kill(-k, signal.SIGINT) 545 os.kill(-k, signal.SIGINT)
501 raise 546 except:
547 pass
548 raise
502 549
503 # Sanity Checks 550 # Sanity Checks
504 for task in range(len(self.runq_fnid)): 551 for task in range(len(self.runq_fnid)):
505 if runq_buildable[task] == 0: 552 if self.runq_buildable[task] == 0:
506 bb.msg.error(bb.msg.domain.RunQueue, "Task %s never buildable!" % task) 553 bb.msg.error(bb.msg.domain.RunQueue, "Task %s never buildable!" % task)
507 if runq_running[task] == 0: 554 if self.runq_running[task] == 0:
508 bb.msg.error(bb.msg.domain.RunQueue, "Task %s never ran!" % task) 555 bb.msg.error(bb.msg.domain.RunQueue, "Task %s never ran!" % task)
509 if runq_complete[task] == 0: 556 if self.runq_complete[task] == 0:
510 bb.msg.error(bb.msg.domain.RunQueue, "Task %s never completed!" % task) 557 bb.msg.error(bb.msg.domain.RunQueue, "Task %s never completed!" % task)
511 558
512 bb.msg.note(1, bb.msg.domain.RunQueue, "Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and %d failed." % (tasks_completed, tasks_skipped, len(failed_fnids))) 559 bb.msg.note(1, bb.msg.domain.RunQueue, "Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and %d failed." % (self.stats.completed, self.stats.skipped, self.stats.failed))
513 560
514 return failed_fnids 561 return self.failed_fnids
515 562
516 def dump_data(self, taskQueue): 563 def dump_data(self, taskQueue):
517 """ 564 """
diff --git a/bitbake/lib/bb/shell.py b/bitbake/lib/bb/shell.py
index 32a773064b..fc213c3f4a 100644
--- a/bitbake/lib/bb/shell.py
+++ b/bitbake/lib/bb/shell.py
@@ -104,10 +104,11 @@ class BitBakeShellCommands:
104 104
105 def _findProvider( self, item ): 105 def _findProvider( self, item ):
106 self._checkParsed() 106 self._checkParsed()
107 # Need to use taskData for this information
107 preferred = data.getVar( "PREFERRED_PROVIDER_%s" % item, cooker.configuration.data, 1 ) 108 preferred = data.getVar( "PREFERRED_PROVIDER_%s" % item, cooker.configuration.data, 1 )
108 if not preferred: preferred = item 109 if not preferred: preferred = item
109 try: 110 try:
110 lv, lf, pv, pf = Providers.findBestProvider(preferred, cooker.configuration.data, cooker.status, cooker.build_cache_fail) 111 lv, lf, pv, pf = Providers.findBestProvider(preferred, cooker.configuration.data, cooker.status)
111 except KeyError: 112 except KeyError:
112 if item in cooker.status.providers: 113 if item in cooker.status.providers:
113 pf = cooker.status.providers[item][0] 114 pf = cooker.status.providers[item][0]
@@ -144,6 +145,7 @@ class BitBakeShellCommands:
144 145
145 def build( self, params, cmd = "build" ): 146 def build( self, params, cmd = "build" ):
146 """Build a providee""" 147 """Build a providee"""
148 global last_exception
147 globexpr = params[0] 149 globexpr = params[0]
148 self._checkParsed() 150 self._checkParsed()
149 names = globfilter( cooker.status.pkg_pn.keys(), globexpr ) 151 names = globfilter( cooker.status.pkg_pn.keys(), globexpr )
@@ -152,8 +154,6 @@ class BitBakeShellCommands:
152 154
153 oldcmd = cooker.configuration.cmd 155 oldcmd = cooker.configuration.cmd
154 cooker.configuration.cmd = cmd 156 cooker.configuration.cmd = cmd
155 cooker.build_cache = []
156 cooker.build_cache_fail = []
157 157
158 td = taskdata.TaskData(cooker.configuration.abort) 158 td = taskdata.TaskData(cooker.configuration.abort)
159 159
@@ -170,24 +170,21 @@ class BitBakeShellCommands:
170 170
171 td.add_unresolved(cooker.configuration.data, cooker.status) 171 td.add_unresolved(cooker.configuration.data, cooker.status)
172 172
173 rq = runqueue.RunQueue() 173 rq = runqueue.RunQueue(cooker, cooker.configuration.data, cooker.status, td, tasks)
174 rq.prepare_runqueue(cooker, cooker.configuration.data, cooker.status, td, tasks) 174 rq.prepare_runqueue()
175 rq.execute_runqueue(cooker, cooker.configuration.data, cooker.status, td, tasks) 175 rq.execute_runqueue()
176 176
177 except Providers.NoProvider: 177 except Providers.NoProvider:
178 print "ERROR: No Provider" 178 print "ERROR: No Provider"
179 global last_exception
180 last_exception = Providers.NoProvider 179 last_exception = Providers.NoProvider
181 180
182 except runqueue.TaskFailure, fnids: 181 except runqueue.TaskFailure, fnids:
183 for fnid in fnids: 182 for fnid in fnids:
184 print "ERROR: '%s' failed" % td.fn_index[fnid] 183 print "ERROR: '%s' failed" % td.fn_index[fnid]
185 global last_exception
186 last_exception = runqueue.TaskFailure 184 last_exception = runqueue.TaskFailure
187 185
188 except build.EventException, e: 186 except build.EventException, e:
189 print "ERROR: Couldn't build '%s'" % names 187 print "ERROR: Couldn't build '%s'" % names
190 global last_exception
191 last_exception = e 188 last_exception = e
192 189
193 cooker.configuration.cmd = oldcmd 190 cooker.configuration.cmd = oldcmd
@@ -236,14 +233,13 @@ class BitBakeShellCommands:
236 233
237 def fileBuild( self, params, cmd = "build" ): 234 def fileBuild( self, params, cmd = "build" ):
238 """Parse and build a .bb file""" 235 """Parse and build a .bb file"""
236 global last_exception
239 name = params[0] 237 name = params[0]
240 bf = completeFilePath( name ) 238 bf = completeFilePath( name )
241 print "SHELL: Calling '%s' on '%s'" % ( cmd, bf ) 239 print "SHELL: Calling '%s' on '%s'" % ( cmd, bf )
242 240
243 oldcmd = cooker.configuration.cmd 241 oldcmd = cooker.configuration.cmd
244 cooker.configuration.cmd = cmd 242 cooker.configuration.cmd = cmd
245 cooker.build_cache = []
246 cooker.build_cache_fail = []
247 243
248 thisdata = copy.deepcopy( initdata ) 244 thisdata = copy.deepcopy( initdata )
249 # Caution: parse.handle modifies thisdata, hence it would 245 # Caution: parse.handle modifies thisdata, hence it would
@@ -266,7 +262,6 @@ class BitBakeShellCommands:
266 cooker.tryBuildPackage( os.path.abspath( bf ), item, cmd, bbfile_data, True ) 262 cooker.tryBuildPackage( os.path.abspath( bf ), item, cmd, bbfile_data, True )
267 except build.EventException, e: 263 except build.EventException, e:
268 print "ERROR: Couldn't build '%s'" % name 264 print "ERROR: Couldn't build '%s'" % name
269 global last_exception
270 last_exception = e 265 last_exception = e
271 266
272 cooker.configuration.cmd = oldcmd 267 cooker.configuration.cmd = oldcmd
@@ -537,8 +532,6 @@ SRC_URI = ""
537 def status( self, params ): 532 def status( self, params ):
538 """<just for testing>""" 533 """<just for testing>"""
539 print "-" * 78 534 print "-" * 78
540 print "build cache = '%s'" % cooker.build_cache
541 print "build cache fail = '%s'" % cooker.build_cache_fail
542 print "building list = '%s'" % cooker.building_list 535 print "building list = '%s'" % cooker.building_list
543 print "build path = '%s'" % cooker.build_path 536 print "build path = '%s'" % cooker.build_path
544 print "consider_msgs_cache = '%s'" % cooker.consider_msgs_cache 537 print "consider_msgs_cache = '%s'" % cooker.consider_msgs_cache
@@ -557,6 +550,7 @@ SRC_URI = ""
557 550
558 def which( self, params ): 551 def which( self, params ):
559 """Computes the providers for a given providee""" 552 """Computes the providers for a given providee"""
553 # Need to use taskData for this information
560 item = params[0] 554 item = params[0]
561 555
562 self._checkParsed() 556 self._checkParsed()
@@ -565,8 +559,7 @@ SRC_URI = ""
565 if not preferred: preferred = item 559 if not preferred: preferred = item
566 560
567 try: 561 try:
568 lv, lf, pv, pf = Providers.findBestProvider(preferred, cooker.configuration.data, cooker.status, 562 lv, lf, pv, pf = Providers.findBestProvider(preferred, cooker.configuration.data, cooker.status)
569cooker.build_cache_fail)
570 except KeyError: 563 except KeyError:
571 lv, lf, pv, pf = (None,)*4 564 lv, lf, pv, pf = (None,)*4
572 565
diff --git a/bitbake/lib/bb/taskdata.py b/bitbake/lib/bb/taskdata.py
index 17d6d95530..3d3adfdbda 100644
--- a/bitbake/lib/bb/taskdata.py
+++ b/bitbake/lib/bb/taskdata.py
@@ -43,6 +43,7 @@ class TaskData:
43 self.tasks_fnid = [] 43 self.tasks_fnid = []
44 self.tasks_name = [] 44 self.tasks_name = []
45 self.tasks_tdepends = [] 45 self.tasks_tdepends = []
46 self.tasks_idepends = []
46 # Cache to speed up task ID lookups 47 # Cache to speed up task ID lookups
47 self.tasks_lookup = {} 48 self.tasks_lookup = {}
48 49
@@ -108,6 +109,7 @@ class TaskData:
108 self.tasks_name.append(task) 109 self.tasks_name.append(task)
109 self.tasks_fnid.append(fnid) 110 self.tasks_fnid.append(fnid)
110 self.tasks_tdepends.append([]) 111 self.tasks_tdepends.append([])
112 self.tasks_idepends.append([])
111 113
112 listid = len(self.tasks_name) - 1 114 listid = len(self.tasks_name) - 1
113 115
@@ -134,8 +136,9 @@ class TaskData:
134 if fnid in self.tasks_fnid: 136 if fnid in self.tasks_fnid:
135 return 137 return
136 138
137 # Work out task dependencies
138 for task in task_graph.allnodes(): 139 for task in task_graph.allnodes():
140
141 # Work out task dependencies
139 parentids = [] 142 parentids = []
140 for dep in task_graph.getparents(task): 143 for dep in task_graph.getparents(task):
141 parentid = self.gettask_id(fn, dep) 144 parentid = self.gettask_id(fn, dep)
@@ -143,6 +146,14 @@ class TaskData:
143 taskid = self.gettask_id(fn, task) 146 taskid = self.gettask_id(fn, task)
144 self.tasks_tdepends[taskid].extend(parentids) 147 self.tasks_tdepends[taskid].extend(parentids)
145 148
149 # Touch all intertask dependencies
150 if 'depends' in task_deps and task in task_deps['depends']:
151 ids = []
152 for dep in task_deps['depends'][task].split(" "):
153 if dep:
154 ids.append(str(self.getbuild_id(dep.split(":")[0])) + ":" + dep.split(":")[1])
155 self.tasks_idepends[taskid].extend(ids)
156
146 # Work out build dependencies 157 # Work out build dependencies
147 if not fnid in self.depids: 158 if not fnid in self.depids:
148 dependids = {} 159 dependids = {}
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index 411f43a105..c2884f2633 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -62,10 +62,12 @@ def vercmp_part(a, b):
62 return -1 62 return -1
63 63
64def vercmp(ta, tb): 64def vercmp(ta, tb):
65 (va, ra) = ta 65 (ea, va, ra) = ta
66 (vb, rb) = tb 66 (eb, vb, rb) = tb
67 67
68 r = vercmp_part(va, vb) 68 r = int(ea)-int(eb)
69 if (r == 0):
70 r = vercmp_part(va, vb)
69 if (r == 0): 71 if (r == 0):
70 r = vercmp_part(ra, rb) 72 r = vercmp_part(ra, rb)
71 return r 73 return r
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2024-05-04 22:17:08 +0000