summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorRichard Purdie <richard@openedhand.com>2006-11-16 15:02:15 (GMT)
committerRichard Purdie <richard@openedhand.com>2006-11-16 15:02:15 (GMT)
commit306b7c7a9757ead077363074e7bbac2e5c03e7c5 (patch)
tree6935017a9af749c46816881c86258f514384ba1c /bitbake
parent65930a38e415ae4a0182e1cea1be838e0ada50ee (diff)
downloadpoky-306b7c7a9757ead077363074e7bbac2e5c03e7c5.tar.gz
bitbake: Upgrade from 1.4 -> 1.7.4ish
git-svn-id: https://svn.o-hand.com/repos/poky/trunk@863 311d38ba-8fff-0310-9ca6-ca027cbcb966
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/ChangeLog18
-rw-r--r--bitbake/MANIFEST17
-rw-r--r--bitbake/TODO18
-rwxr-xr-xbitbake/bin/bbimage5
-rwxr-xr-xbitbake/bin/bitbake892
-rwxr-xr-xbitbake/bin/bitdoc2
-rw-r--r--bitbake/classes/base.bbclass2
-rw-r--r--bitbake/conf/bitbake.conf5
-rw-r--r--bitbake/contrib/vim/ftdetect/bitbake.vim4
-rw-r--r--bitbake/contrib/vim/syntax/bitbake.vim8
-rw-r--r--bitbake/doc/manual/usermanual.xml29
-rw-r--r--bitbake/lib/bb/COW.py305
-rw-r--r--bitbake/lib/bb/__init__.py27
-rw-r--r--bitbake/lib/bb/build.py110
-rw-r--r--bitbake/lib/bb/cache.py207
-rw-r--r--bitbake/lib/bb/data.py49
-rw-r--r--bitbake/lib/bb/data_smart.py43
-rw-r--r--bitbake/lib/bb/fetch/__init__.py167
-rw-r--r--bitbake/lib/bb/fetch/cvs.py255
-rw-r--r--bitbake/lib/bb/fetch/git.py144
-rw-r--r--bitbake/lib/bb/fetch/local.py18
-rw-r--r--bitbake/lib/bb/fetch/perforce.py213
-rw-r--r--bitbake/lib/bb/fetch/ssh.py94
-rw-r--r--bitbake/lib/bb/fetch/svk.py160
-rw-r--r--bitbake/lib/bb/fetch/svn.py203
-rw-r--r--bitbake/lib/bb/fetch/wget.py152
-rw-r--r--bitbake/lib/bb/methodpool.py3
-rw-r--r--bitbake/lib/bb/msg.py108
-rw-r--r--bitbake/lib/bb/parse/__init__.py11
-rw-r--r--bitbake/lib/bb/parse/parse_c/BBHandler.py151
-rw-r--r--bitbake/lib/bb/parse/parse_c/Makefile16
-rw-r--r--bitbake/lib/bb/parse/parse_c/bitbakec.pyx205
-rw-r--r--bitbake/lib/bb/parse/parse_c/bitbakeparser.cc144
-rw-r--r--bitbake/lib/bb/parse/parse_c/bitbakescanner.cc218
-rw-r--r--bitbake/lib/bb/parse/parse_c/bitbakescanner.l30
-rw-r--r--bitbake/lib/bb/parse/parse_c/lexer.h8
-rw-r--r--bitbake/lib/bb/parse/parse_c/lexerc.h2
-rw-r--r--bitbake/lib/bb/parse/parse_py/BBHandler.py28
-rw-r--r--bitbake/lib/bb/parse/parse_py/ConfHandler.py33
-rw-r--r--bitbake/lib/bb/providers.py209
-rw-r--r--bitbake/lib/bb/runqueue.py491
-rw-r--r--bitbake/lib/bb/shell.py113
-rw-r--r--bitbake/lib/bb/taskdata.py558
-rw-r--r--bitbake/lib/bb/utils.py54
44 files changed, 3720 insertions, 1809 deletions
diff --git a/bitbake/ChangeLog b/bitbake/ChangeLog
index 93cc45a..135aba9 100644
--- a/bitbake/ChangeLog
+++ b/bitbake/ChangeLog
@@ -1,3 +1,21 @@
1Changes in BitBake 1.7.3:
2
3Changes in BitBake 1.7.1:
4 - Major updates of the dependency handling and execution
5 of tasks
6 - Change of the SVN Fetcher to keep the checkout around
7 courtsey to Paul Sokolovsky (#1367)
8
9Changes in Bitbake 1.6.0:
10 - Better msg handling
11 - COW dict implementation from Tim Ansell (mithro) leading
12 to better performance
13 - Speed up of -s
14
15Changes in Bitbake 1.4.4:
16 - SRCDATE now handling courtsey Justin Patrin
17 - #1017 fix to work with rm_work
18
1Changes in BitBake 1.4.2: 19Changes in BitBake 1.4.2:
2 - Send logs to oe.pastebin.com instead of pastebin.com 20 - Send logs to oe.pastebin.com instead of pastebin.com
3 fixes #856 21 fixes #856
diff --git a/bitbake/MANIFEST b/bitbake/MANIFEST
index 144c74c..f088792 100644
--- a/bitbake/MANIFEST
+++ b/bitbake/MANIFEST
@@ -5,29 +5,34 @@ setup.py
5bin/bitdoc 5bin/bitdoc
6bin/bbimage 6bin/bbimage
7bin/bitbake 7bin/bitbake
8lib/bb/COW.py
8lib/bb/__init__.py 9lib/bb/__init__.py
9lib/bb/build.py 10lib/bb/build.py
10lib/bb/cache.py 11lib/bb/cache.py
11lib/bb/data.py 12lib/bb/data.py
12lib/bb/data_smart.py 13lib/bb/data_smart.py
13lib/bb/event.py 14lib/bb/event.py
14lib/bb/fetch/bk.py 15lib/bb/manifest.py
16lib/bb/methodpool.py
17lib/bb/msg.py
18lib/bb/providers.py
19lib/bb/runqueue.py
20lib/bb/shell.py
21lib/bb/taskdata.py
22lib/bb/utils.py
15lib/bb/fetch/cvs.py 23lib/bb/fetch/cvs.py
16lib/bb/fetch/git.py 24lib/bb/fetch/git.py
17lib/bb/fetch/__init__.py 25lib/bb/fetch/__init__.py
18lib/bb/fetch/local.py 26lib/bb/fetch/local.py
27lib/bb/fetch/perforce.py
28lib/bb/fetch/ssh.py
19lib/bb/fetch/svk.py 29lib/bb/fetch/svk.py
20lib/bb/fetch/svn.py 30lib/bb/fetch/svn.py
21lib/bb/fetch/wget.py 31lib/bb/fetch/wget.py
22lib/bb/fetch/ssh.py
23lib/bb/manifest.py
24lib/bb/methodpool.py
25lib/bb/parse/__init__.py 32lib/bb/parse/__init__.py
26lib/bb/parse/parse_py/BBHandler.py 33lib/bb/parse/parse_py/BBHandler.py
27lib/bb/parse/parse_py/ConfHandler.py 34lib/bb/parse/parse_py/ConfHandler.py
28lib/bb/parse/parse_py/__init__.py 35lib/bb/parse/parse_py/__init__.py
29lib/bb/shell.py
30lib/bb/utils.py
31doc/COPYING.GPL 36doc/COPYING.GPL
32doc/COPYING.MIT 37doc/COPYING.MIT
33doc/manual/html.css 38doc/manual/html.css
diff --git a/bitbake/TODO b/bitbake/TODO
deleted file mode 100644
index 511fae4..0000000
--- a/bitbake/TODO
+++ /dev/null
@@ -1,18 +0,0 @@
1On popular request by popular people a list of tasks to-do:
2
3 -Kill insecure usage of os.system either by properly escaping
4 the strings or a faster replacement not involving /bin/sh
5 -Introduce a -p option to automatically hotshot/profile the
6 run
7 -Cache dependencies separately and invalidate them when any file
8 changed.
9 -...
10
11
12DONE:
13 -On generating the inter package deps do not parse each file multiply
14 times.
15 -We build the lists while parsing the data now
16 (WAS: Do not generate the world dependency tree, only when someone
17 requests it.
18
diff --git a/bitbake/bin/bbimage b/bitbake/bin/bbimage
index df6caa2..9adedbf 100755
--- a/bitbake/bin/bbimage
+++ b/bitbake/bin/bbimage
@@ -18,15 +18,16 @@
18# Place, Suite 330, Boston, MA 02111-1307 USA. 18# Place, Suite 330, Boston, MA 02111-1307 USA.
19 19
20import sys, os 20import sys, os
21sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) 21sys.path.insert(0,os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
22import bb 22import bb
23from bb import * 23from bb import *
24 24
25__version__ = 1.0 25__version__ = 1.1
26type = "jffs2" 26type = "jffs2"
27cfg_bb = data.init() 27cfg_bb = data.init()
28cfg_oespawn = data.init() 28cfg_oespawn = data.init()
29 29
30bb.msg.set_debug_level(0)
30 31
31def usage(): 32def usage():
32 print "Usage: bbimage [options ...]" 33 print "Usage: bbimage [options ...]"
diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake
index 7fbe7ed..85a0cbc 100755
--- a/bitbake/bin/bitbake
+++ b/bitbake/bin/bitbake
@@ -7,6 +7,7 @@
7# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer 7# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8# Copyright (C) 2005 Holger Hans Peter Freyther 8# Copyright (C) 2005 Holger Hans Peter Freyther
9# Copyright (C) 2005 ROAD GmbH 9# Copyright (C) 2005 ROAD GmbH
10# Copyright (C) 2006 Richard Purdie
10# 11#
11# This program is free software; you can redistribute it and/or modify it under 12# This program is free software; you can redistribute it and/or modify it under
12# the terms of the GNU General Public License as published by the Free Software 13# the terms of the GNU General Public License as published by the Free Software
@@ -24,136 +25,13 @@
24import sys, os, getopt, glob, copy, os.path, re, time 25import sys, os, getopt, glob, copy, os.path, re, time
25sys.path.insert(0,os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) 26sys.path.insert(0,os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
26import bb 27import bb
27from bb import utils, data, parse, debug, event, fatal, cache 28from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
28from sets import Set 29from sets import Set
29import itertools, optparse 30import itertools, optparse
30 31
31parsespin = itertools.cycle( r'|/-\\' ) 32parsespin = itertools.cycle( r'|/-\\' )
32bbdebug = 0
33
34__version__ = "1.4.3"
35
36#============================================================================#
37# BBParsingStatus
38#============================================================================#
39class BBParsingStatus:
40 """
41 The initial idea for this status class is to use the data when it is
42 already loaded instead of loading it from various place over and over
43 again.
44 """
45
46 def __init__(self):
47 self.providers = {}
48 self.rproviders = {}
49 self.packages = {}
50 self.packages_dynamic = {}
51 self.bbfile_priority = {}
52 self.bbfile_config_priorities = []
53 self.ignored_dependencies = None
54 self.possible_world = []
55 self.world_target = Set()
56 self.pkg_pn = {}
57 self.pkg_fn = {}
58 self.pkg_pvpr = {}
59 self.pkg_dp = {}
60 self.pn_provides = {}
61 self.all_depends = Set()
62 self.build_all = {}
63 self.rundeps = {}
64 self.runrecs = {}
65 self.stamp = {}
66
67 def handle_bb_data(self, file_name, bb_cache, cached):
68 """
69 We will fill the dictionaries with the stuff we
70 need for building the tree more fast
71 """
72
73 pn = bb_cache.getVar('PN', file_name, True)
74 pv = bb_cache.getVar('PV', file_name, True)
75 pr = bb_cache.getVar('PR', file_name, True)
76 dp = int(bb_cache.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
77 provides = Set([pn] + (bb_cache.getVar("PROVIDES", file_name, True) or "").split())
78 depends = (bb_cache.getVar("DEPENDS", file_name, True) or "").split()
79 packages = (bb_cache.getVar('PACKAGES', file_name, True) or "").split()
80 packages_dynamic = (bb_cache.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
81 rprovides = (bb_cache.getVar("RPROVIDES", file_name, True) or "").split()
82
83 # build PackageName to FileName lookup table
84 if pn not in self.pkg_pn:
85 self.pkg_pn[pn] = []
86 self.pkg_pn[pn].append(file_name)
87
88 self.build_all[file_name] = int(bb_cache.getVar('BUILD_ALL_DEPS', file_name, True) or "0")
89 self.stamp[file_name] = bb_cache.getVar('STAMP', file_name, True)
90
91 # build FileName to PackageName lookup table
92 self.pkg_fn[file_name] = pn
93 self.pkg_pvpr[file_name] = (pv,pr)
94 self.pkg_dp[file_name] = dp
95
96 # Build forward and reverse provider hashes
97 # Forward: virtual -> [filenames]
98 # Reverse: PN -> [virtuals]
99 if pn not in self.pn_provides:
100 self.pn_provides[pn] = Set()
101 self.pn_provides[pn] |= provides
102
103 for provide in provides:
104 if provide not in self.providers:
105 self.providers[provide] = []
106 self.providers[provide].append(file_name)
107
108 for dep in depends:
109 self.all_depends.add(dep)
110
111 # Build reverse hash for PACKAGES, so runtime dependencies
112 # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
113 for package in packages:
114 if not package in self.packages:
115 self.packages[package] = []
116 self.packages[package].append(file_name)
117 rprovides += (bb_cache.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split()
118
119 for package in packages_dynamic:
120 if not package in self.packages_dynamic:
121 self.packages_dynamic[package] = []
122 self.packages_dynamic[package].append(file_name)
123
124 for rprovide in rprovides:
125 if not rprovide in self.rproviders:
126 self.rproviders[rprovide] = []
127 self.rproviders[rprovide].append(file_name)
128
129 # Build hash of runtime depeneds and rececommends
130
131 def add_dep(deplist, deps):
132 for dep in deps:
133 if not dep in deplist:
134 deplist[dep] = ""
135
136 if not file_name in self.rundeps:
137 self.rundeps[file_name] = {}
138 if not file_name in self.runrecs:
139 self.runrecs[file_name] = {}
140
141 for package in packages + [pn]:
142 if not package in self.rundeps[file_name]:
143 self.rundeps[file_name][package] = {}
144 if not package in self.runrecs[file_name]:
145 self.runrecs[file_name][package] = {}
146
147 add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RDEPENDS', file_name, True) or ""))
148 add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RRECOMMENDS', file_name, True) or ""))
149 add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RDEPENDS_%s" % package, file_name, True) or ""))
150 add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RRECOMMENDS_%s" % package, file_name, True) or ""))
151
152 # Collect files we may need for possible world-dep
153 # calculations
154 if not bb_cache.getVar('BROKEN', file_name, True) and not bb_cache.getVar('EXCLUDE_FROM_WORLD', file_name, True):
155 self.possible_world.append(file_name)
156 33
34__version__ = "1.7.4"
157 35
158#============================================================================# 36#============================================================================#
159# BBStatistics 37# BBStatistics
@@ -198,207 +76,63 @@ class BBCooker:
198 Manages one bitbake build run 76 Manages one bitbake build run
199 """ 77 """
200 78
201 ParsingStatus = BBParsingStatus # make it visible from the shell
202 Statistics = BBStatistics # make it visible from the shell 79 Statistics = BBStatistics # make it visible from the shell
203 80
204 def __init__( self ): 81 def __init__( self ):
205 self.build_cache_fail = [] 82 self.build_cache_fail = []
206 self.build_cache = [] 83 self.build_cache = []
207 self.rbuild_cache = []
208 self.building_list = []
209 self.build_path = []
210 self.consider_msgs_cache = []
211 self.preferred = {}
212 self.stats = BBStatistics() 84 self.stats = BBStatistics()
213 self.status = None 85 self.status = None
214 86
215 self.cache = None 87 self.cache = None
216 self.bb_cache = None 88 self.bb_cache = None
217 89
218 def tryBuildPackage( self, fn, item, the_data ): 90 def tryBuildPackage(self, fn, item, task, the_data, build_depends):
219 """Build one package""" 91 """
92 Build one task of a package, optionally build following task depends
93 """
220 bb.event.fire(bb.event.PkgStarted(item, the_data)) 94 bb.event.fire(bb.event.PkgStarted(item, the_data))
221 try: 95 try:
222 self.stats.attempt += 1 96 self.stats.attempt += 1
223 if self.configuration.force: 97 if self.configuration.force:
224 bb.data.setVarFlag('do_%s' % self.configuration.cmd, 'force', 1, the_data) 98 bb.data.setVarFlag('do_%s' % task, 'force', 1, the_data)
99 if not build_depends:
100 bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data)
225 if not self.configuration.dry_run: 101 if not self.configuration.dry_run:
226 bb.build.exec_task('do_%s' % self.configuration.cmd, the_data) 102 bb.build.exec_task('do_%s' % task, the_data)
227 bb.event.fire(bb.event.PkgSucceeded(item, the_data)) 103 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
228 self.build_cache.append(fn) 104 self.build_cache.append(fn)
229 return True 105 return True
230 except bb.build.FuncFailed: 106 except bb.build.FuncFailed:
231 self.stats.fail += 1 107 self.stats.fail += 1
232 bb.error("task stack execution failed") 108 bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
233 bb.event.fire(bb.event.PkgFailed(item, the_data)) 109 bb.event.fire(bb.event.PkgFailed(item, the_data))
234 self.build_cache_fail.append(fn) 110 self.build_cache_fail.append(fn)
235 raise 111 raise
236 except bb.build.EventException, e: 112 except bb.build.EventException, e:
237 self.stats.fail += 1 113 self.stats.fail += 1
238 event = e.args[1] 114 event = e.args[1]
239 bb.error("%s event exception, aborting" % bb.event.getName(event)) 115 bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
240 bb.event.fire(bb.event.PkgFailed(item, the_data)) 116 bb.event.fire(bb.event.PkgFailed(item, the_data))
241 self.build_cache_fail.append(fn) 117 self.build_cache_fail.append(fn)
242 raise 118 raise
243 119
244 def tryBuild( self, fn, virtual , buildAllDeps , build_depends = []): 120 def tryBuild( self, fn, build_depends):
245 """ 121 """
246 Build a provider and its dependencies. 122 Build a provider and its dependencies.
247 build_depends is a list of previous build dependencies (not runtime) 123 build_depends is a list of previous build dependencies (not runtime)
248 If build_depends is empty, we're dealing with a runtime depends 124 If build_depends is empty, we're dealing with a runtime depends
249 """ 125 """
250 126
251 the_data = self.bb_cache.loadDataFull(fn, self) 127 the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
252
253 # Only follow all (runtime) dependencies if doing a build
254 if not buildAllDeps and self.configuration.cmd is "build":
255 buildAllDeps = self.status.build_all[fn]
256
257 # Error on build time dependency loops
258 if build_depends and build_depends.count(fn) > 1:
259 bb.error("%s depends on itself (eventually)" % fn)
260 bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
261 return False
262
263 # See if this is a runtime dependency we've already built
264 # Or a build dependency being handled in a different build chain
265 if fn in self.building_list:
266 return self.addRunDeps(fn, virtual , buildAllDeps)
267 128
268 item = self.status.pkg_fn[fn] 129 item = self.status.pkg_fn[fn]
269 130
270 self.building_list.append(fn) 131 if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data) and not self.configuration.force:
271 132 self.build_cache.append(fn)
272 pathstr = "%s (%s)" % (item, virtual) 133 return True
273 self.build_path.append(pathstr)
274
275 depends_list = (bb.data.getVar('DEPENDS', the_data, True) or "").split()
276
277 if self.configuration.verbose:
278 bb.note("current path: %s" % (" -> ".join(self.build_path)))
279 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
280
281 try:
282 failed = False
283
284 depcmd = self.configuration.cmd
285 bbdepcmd = bb.data.getVarFlag('do_%s' % self.configuration.cmd, 'bbdepcmd', the_data)
286 if bbdepcmd is not None:
287 if bbdepcmd == "":
288 depcmd = None
289 else:
290 depcmd = bbdepcmd
291
292 if depcmd:
293 oldcmd = self.configuration.cmd
294 self.configuration.cmd = depcmd
295
296 for dependency in depends_list:
297 if dependency in self.status.ignored_dependencies:
298 continue
299 if not depcmd:
300 continue
301 if self.buildProvider( dependency , buildAllDeps , build_depends ) == 0:
302 bb.error("dependency %s (for %s) not satisfied" % (dependency,item))
303 failed = True
304 if self.configuration.abort:
305 break
306
307 if depcmd:
308 self.configuration.cmd = oldcmd
309
310 if failed:
311 self.stats.deps += 1
312 return False
313
314 if not self.addRunDeps(fn, virtual , buildAllDeps):
315 return False
316
317 if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
318 self.build_cache.append(fn)
319 return True
320
321 return self.tryBuildPackage( fn, item, the_data )
322
323 finally:
324 self.building_list.remove(fn)
325 self.build_path.remove(pathstr)
326
327 def findBestProvider( self, pn, pkg_pn = None):
328 """
329 If there is a PREFERRED_VERSION, find the highest-priority bbfile
330 providing that version. If not, find the latest version provided by
331 an bbfile in the highest-priority set.
332 """
333 if not pkg_pn:
334 pkg_pn = self.status.pkg_pn
335
336 files = pkg_pn[pn]
337 priorities = {}
338 for f in files:
339 priority = self.status.bbfile_priority[f]
340 if priority not in priorities:
341 priorities[priority] = []
342 priorities[priority].append(f)
343 p_list = priorities.keys()
344 p_list.sort(lambda a, b: a - b)
345 tmp_pn = []
346 for p in p_list:
347 tmp_pn = [priorities[p]] + tmp_pn
348
349 preferred_file = None
350
351 localdata = data.createCopy(self.configuration.data)
352 bb.data.setVar('OVERRIDES', "%s:%s" % (pn, data.getVar('OVERRIDES', localdata)), localdata)
353 bb.data.update_data(localdata)
354
355 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, localdata, True)
356 if preferred_v:
357 m = re.match('(.*)_(.*)', preferred_v)
358 if m:
359 preferred_v = m.group(1)
360 preferred_r = m.group(2)
361 else:
362 preferred_r = None
363
364 for file_set in tmp_pn:
365 for f in file_set:
366 pv,pr = self.status.pkg_pvpr[f]
367 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
368 preferred_file = f
369 preferred_ver = (pv, pr)
370 break
371 if preferred_file:
372 break;
373 if preferred_r:
374 pv_str = '%s-%s' % (preferred_v, preferred_r)
375 else:
376 pv_str = preferred_v
377 if preferred_file is None:
378 bb.note("preferred version %s of %s not available" % (pv_str, pn))
379 else:
380 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
381
382 del localdata
383
384 # get highest priority file set
385 files = tmp_pn[0]
386 latest = None
387 latest_p = 0
388 latest_f = None
389 for file_name in files:
390 pv,pr = self.status.pkg_pvpr[file_name]
391 dp = self.status.pkg_dp[file_name]
392
393 if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
394 latest = (pv, pr)
395 latest_f = file_name
396 latest_p = dp
397 if preferred_file is None:
398 preferred_file = latest_f
399 preferred_ver = latest
400 134
401 return (latest,latest_f,preferred_ver, preferred_file) 135 return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends)
402 136
403 def showVersions( self ): 137 def showVersions( self ):
404 pkg_pn = self.status.pkg_pn 138 pkg_pn = self.status.pkg_pn
@@ -407,7 +141,7 @@ class BBCooker:
407 141
408 # Sort by priority 142 # Sort by priority
409 for pn in pkg_pn.keys(): 143 for pn in pkg_pn.keys():
410 (last_ver,last_file,pref_ver,pref_file) = self.findBestProvider(pn) 144 (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
411 preferred_versions[pn] = (pref_ver, pref_file) 145 preferred_versions[pn] = (pref_ver, pref_file)
412 latest_versions[pn] = (last_ver, last_file) 146 latest_versions[pn] = (last_ver, last_file)
413 147
@@ -425,7 +159,7 @@ class BBCooker:
425 159
426 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1], 160 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
427 prefstr) 161 prefstr)
428 162
429 163
430 def showEnvironment( self ): 164 def showEnvironment( self ):
431 """Show the outer or per-package environment""" 165 """Show the outer or per-package environment"""
@@ -433,268 +167,190 @@ class BBCooker:
433 self.cb = None 167 self.cb = None
434 self.bb_cache = bb.cache.init(self) 168 self.bb_cache = bb.cache.init(self)
435 try: 169 try:
436 self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self) 170 self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self.configuration.data)
437 except IOError, e: 171 except IOError, e:
438 fatal("Unable to read %s: %s" % ( self.configuration.buildfile, e )) 172 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % ( self.configuration.buildfile, e ))
439 except Exception, e: 173 except Exception, e:
440 fatal("%s" % e) 174 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
441 # emit variables and shell functions 175 # emit variables and shell functions
442 try: 176 try:
443 data.update_data( self.configuration.data ) 177 data.update_data( self.configuration.data )
444 data.emit_env(sys.__stdout__, self.configuration.data, True) 178 data.emit_env(sys.__stdout__, self.configuration.data, True)
445 except Exception, e: 179 except Exception, e:
446 fatal("%s" % e) 180 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
447 # emit the metadata which isnt valid shell 181 # emit the metadata which isnt valid shell
182 data.expandKeys( self.configuration.data )
448 for e in self.configuration.data.keys(): 183 for e in self.configuration.data.keys():
449 if data.getVarFlag( e, 'python', self.configuration.data ): 184 if data.getVarFlag( e, 'python', self.configuration.data ):
450 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1))) 185 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
451 186
452 def filterProviders(self, providers, item): 187 def generateDotGraph( self, pkgs_to_build, ignore_deps ):
453 """ 188 """
454 Take a list of providers and filter/reorder according to the 189 Generate two graphs one for the DEPENDS and RDEPENDS. The current
455 environment variables and previous build results 190 implementation creates crappy graphs ;)
456 """
457 eligible = []
458 preferred_versions = {}
459
460 # Collate providers by PN
461 pkg_pn = {}
462 for p in providers:
463 pn = self.status.pkg_fn[p]
464 if pn not in pkg_pn:
465 pkg_pn[pn] = []
466 pkg_pn[pn].append(p)
467
468 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
469
470 for pn in pkg_pn.keys():
471 preferred_versions[pn] = self.findBestProvider(pn, pkg_pn)[2:4]
472 eligible.append(preferred_versions[pn][1])
473
474 for p in eligible:
475 if p in self.build_cache_fail:
476 bb.debug(1, "rejecting already-failed %s" % p)
477 eligible.remove(p)
478
479 if len(eligible) == 0:
480 bb.error("no eligible providers for %s" % item)
481 return 0
482 191
483 # look to see if one of them is already staged, or marked as preferred. 192 pkgs_to_build A list of packages that needs to be built
484 # if so, bump it to the head of the queue 193 ignore_deps A list of names where processing of dependencies
485 for p in providers: 194 should be stopped. e.g. dependencies that get
486 pn = self.status.pkg_fn[p]
487 pv, pr = self.status.pkg_pvpr[p]
488
489 stamp = '%s.do_populate_staging' % self.status.stamp[p]
490 if os.path.exists(stamp):
491 (newvers, fn) = preferred_versions[pn]
492 if not fn in eligible:
493 # package was made ineligible by already-failed check
494 continue
495 oldver = "%s-%s" % (pv, pr)
496 newver = '-'.join(newvers)
497 if (newver != oldver):
498 extra_chat = "%s (%s) already staged but upgrading to %s to satisfy %s" % (pn, oldver, newver, item)
499 else:
500 extra_chat = "Selecting already-staged %s (%s) to satisfy %s" % (pn, oldver, item)
501 if self.configuration.verbose:
502 bb.note("%s" % extra_chat)
503 eligible.remove(fn)
504 eligible = [fn] + eligible
505 discriminated = True
506 break
507
508 return eligible
509
510 def buildProvider( self, item , buildAllDeps , build_depends = [] ):
511 """ 195 """
512 Build something to provide a named build requirement
513 (takes item names from DEPENDS namespace)
514 """
515
516 fn = None
517 discriminated = False
518
519 if not item in self.status.providers:
520 bb.error("Nothing provides dependency %s" % item)
521 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
522 return 0
523 196
524 all_p = self.status.providers[item] 197 def myFilterProvider( providers, item):
525 198 """
526 for p in all_p: 199 Take a list of providers and filter according to environment
527 if p in self.build_cache: 200 variables. In contrast to filterProviders we do not discriminate
528 bb.debug(1, "already built %s in this run\n" % p) 201 and take PREFERRED_PROVIDER into account.
529 return 1 202 """
530 203 eligible = []
531 eligible = self.filterProviders(all_p, item) 204 preferred_versions = {}
205
206 # Collate providers by PN
207 pkg_pn = {}
208 for p in providers:
209 pn = self.status.pkg_fn[p]
210 if pn not in pkg_pn:
211 pkg_pn[pn] = []
212 pkg_pn[pn].append(p)
532 213
533 if not eligible: 214 bb.msg.debug(1, bb.msg.domain.Provider, "providers for %s are: %s" % (item, pkg_pn.keys()))
534 return 0
535 215
536 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, self.configuration.data, 1) 216 for pn in pkg_pn.keys():
537 if prefervar: 217 preferred_versions[pn] = bb.providers.findBestProvider(pn, self.configuration.data, self.status, pkg_pn)[2:4]
538 self.preferred[item] = prefervar 218 eligible.append(preferred_versions[pn][1])
539 219
540 if item in self.preferred:
541 for p in eligible: 220 for p in eligible:
542 pn = self.status.pkg_fn[p] 221 if p in self.build_cache_fail:
543 if self.preferred[item] == pn: 222 bb.msg.debug(1, bb.msg.domain.Provider, "rejecting already-failed %s" % p)
544 if self.configuration.verbose:
545 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
546 eligible.remove(p) 223 eligible.remove(p)
547 eligible = [p] + eligible
548 discriminated = True
549 break
550 224
551 if len(eligible) > 1 and discriminated == False: 225 if len(eligible) == 0:
552 if item not in self.consider_msgs_cache: 226 bb.msg.error(bb.msg.domain.Provider, "no eligible providers for %s" % item)
553 providers_list = [] 227 return 0
554 for fn in eligible:
555 providers_list.append(self.status.pkg_fn[fn])
556 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
557 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
558 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data))
559 self.consider_msgs_cache.append(item)
560 228
229 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, self.configuration.data, 1)
561 230
562 # run through the list until we find one that we can build 231 # try the preferred provider first
563 for fn in eligible: 232 if prefervar:
564 bb.debug(2, "selecting %s to satisfy %s" % (fn, item)) 233 for p in eligible:
565 if self.tryBuild(fn, item, buildAllDeps, build_depends + [fn]): 234 if prefervar == self.status.pkg_fn[p]:
566 return 1 235 bb.msg.note(1, bb.msg.domain.Provider, "Selecting PREFERRED_PROVIDER %s" % prefervar)
236 eligible.remove(p)
237 eligible = [p] + eligible
567 238
568 bb.note("no buildable providers for %s" % item) 239 return eligible
569 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
570 return 0
571 240
572 def buildRProvider( self, item , buildAllDeps ):
573 """
574 Build something to provide a named runtime requirement
575 (takes item names from RDEPENDS/PACKAGES namespace)
576 """
577 241
578 fn = None 242 # try to avoid adding the same rdepends over an over again
579 all_p = [] 243 seen_depends = []
580 discriminated = False 244 seen_rdepends = []
581 245
582 if not buildAllDeps:
583 return True
584 246
585 all_p = self.getProvidersRun(item) 247 def add_depends(package_list):
248 """
249 Add all depends of all packages from this list
250 """
251 for package in package_list:
252 if package in seen_depends or package in ignore_deps:
253 continue
586 254
587 if not all_p: 255 seen_depends.append( package )
588 bb.error("Nothing provides runtime dependency %s" % (item)) 256 if not package in self.status.providers:
589 bb.event.fire(bb.event.NoProvider(item,self.configuration.data,runtime=True)) 257 """
590 return False 258 We have not seen this name -> error in
259 dependency handling
260 """
261 bb.msg.note(1, bb.msg.domain.Depends, "ERROR with provider: %(package)s" % vars() )
262 print >> depends_file, '"%(package)s" -> ERROR' % vars()
263 continue
591 264
592 for p in all_p: 265 # get all providers for this package
593 if p in self.rbuild_cache: 266 providers = self.status.providers[package]
594 bb.debug(2, "Already built %s providing runtime %s\n" % (p,item))
595 return True
596 if p in self.build_cache:
597 bb.debug(2, "Already built %s but adding any further RDEPENDS for %s\n" % (p, item))
598 return self.addRunDeps(p, item , buildAllDeps)
599 267
600 eligible = self.filterProviders(all_p, item) 268 # now let us find the bestProvider for it
601 if not eligible: 269 fn = myFilterProvider(providers, package)[0]
602 return 0
603 270
604 preferred = [] 271 depends = bb.utils.explode_deps(self.bb_cache.getVar('DEPENDS', fn, True) or "")
605 for p in eligible: 272 version = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
606 pn = self.status.pkg_fn[p] 273 add_depends ( depends )
607 provides = self.status.pn_provides[pn]
608 for provide in provides:
609 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, self.configuration.data, 1)
610 if prefervar == pn:
611 if self.configuration.verbose:
612 bb.note("selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
613 eligible.remove(p)
614 eligible = [p] + eligible
615 preferred.append(p)
616
617 if len(eligible) > 1 and len(preferred) == 0:
618 if item not in self.consider_msgs_cache:
619 providers_list = []
620 for fn in eligible:
621 providers_list.append(self.status.pkg_fn[fn])
622 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
623 bb.note("consider defining a PREFERRED_PROVIDER to match runtime %s" % item)
624 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data,runtime=True))
625 self.consider_msgs_cache.append(item)
626
627 if len(preferred) > 1:
628 if item not in self.consider_msgs_cache:
629 providers_list = []
630 for fn in preferred:
631 providers_list.append(self.status.pkg_fn[fn])
632 bb.note("multiple preferred providers are available (%s);" % ", ".join(providers_list))
633 bb.note("consider defining only one PREFERRED_PROVIDER to match runtime %s" % item)
634 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data,runtime=True))
635 self.consider_msgs_cache.append(item)
636
637 # run through the list until we find one that we can build
638 for fn in eligible:
639 bb.debug(2, "selecting %s to satisfy runtime %s" % (fn, item))
640 if self.tryBuild(fn, item, buildAllDeps):
641 return True
642
643 bb.error("No buildable providers for runtime %s" % item)
644 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
645 return False
646
647 def getProvidersRun(self, rdepend):
648 """
649 Return any potential providers of runtime rdepend
650 """
651 rproviders = []
652 274
653 if rdepend in self.status.rproviders: 275 # now create the node
654 rproviders += self.status.rproviders[rdepend] 276 print >> depends_file, '"%(package)s" [label="%(package)s\\n%(version)s"]' % vars()
655 277
656 if rdepend in self.status.packages: 278 depends = filter( (lambda x: x not in ignore_deps), depends )
657 rproviders += self.status.packages[rdepend] 279 for depend in depends:
280 print >> depends_file, '"%(package)s" -> "%(depend)s"' % vars()
658 281
659 if rproviders:
660 return rproviders
661 282
662 # Only search dynamic packages if we can't find anything in other variables 283 def add_all_depends( the_depends, the_rdepends ):
663 for pattern in self.status.packages_dynamic: 284 """
664 regexp = re.compile(pattern) 285 Add both DEPENDS and RDEPENDS. RDEPENDS will get dashed
665 if regexp.match(rdepend): 286 lines
666 rproviders += self.status.packages_dynamic[pattern] 287 """
288 package_list = the_depends + the_rdepends
289 for package in package_list:
290 if package in seen_rdepends or package in ignore_deps:
291 continue
667 292
668 return rproviders 293 seen_rdepends.append( package )
294
295 # Let us find out if the package is a DEPENDS or RDEPENDS
296 # and we will set 'providers' with the avilable providers
297 # for the package.
298 if package in the_depends:
299 if not package in self.status.providers:
300 bb.msg.note(1, bb.msg.domain.Depends, "ERROR with provider: %(package)s" % vars() )
301 print >> alldepends_file, '"%(package)s" -> ERROR' % vars()
302 continue
303
304 providers = self.status.providers[package]
305 elif package in the_rdepends:
306 if len(bb.providers.getRuntimeProviders(self.status, package)) == 0:
307 bb.msg.note(1, bb.msg.domain.Depends, "ERROR with rprovider: %(package)s" % vars() )
308 print >> alldepends_file, '"%(package)s" -> ERROR [style="dashed"]' % vars()
309 continue
310
311 providers = bb.providers.getRuntimeProviders(self.status, package)
312 else:
313 # something went wrong...
314 print "Complete ERROR! %s" % package
315 continue
669 316
670 def addRunDeps(self , fn, item , buildAllDeps): 317 # now let us find the bestProvider for it
671 """ 318 fn = myFilterProvider(providers, package)[0]
672 Add any runtime dependencies of runtime item provided by fn
673 as long as item has't previously been processed by this function.
674 """
675 319
676 if item in self.rbuild_cache: 320 # Now we have a filename let us get the depends and RDEPENDS of it
677 return True 321 depends = bb.utils.explode_deps(self.bb_cache.getVar('DEPENDS', fn, True) or "")
322 if fn in self.status.rundeps and package in self.status.rundeps[fn]:
323 rdepends= self.status.rundeps[fn][package].keys()
324 else:
325 rdepends = []
326 version = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
678 327
679 if not buildAllDeps: 328 # handle all the depends and rdepends of package
680 return True 329 add_all_depends ( depends, rdepends )
681 330
682 rdepends = [] 331 # now create the node using package name
683 self.rbuild_cache.append(item) 332 print >> alldepends_file, '"%(package)s" [label="%(package)s\\n%(version)s"]' % vars()
684 333
685 if fn in self.status.rundeps and item in self.status.rundeps[fn]: 334 # remove the stuff we want to ignore and add the edges
686 rdepends += self.status.rundeps[fn][item].keys() 335 depends = filter( (lambda x: x not in ignore_deps), depends )
687 if fn in self.status.runrecs and item in self.status.runrecs[fn]: 336 rdepends = filter( (lambda x: x not in ignore_deps), rdepends )
688 rdepends += self.status.runrecs[fn][item].keys() 337 for depend in depends:
338 print >> alldepends_file, '"%(package)s" -> "%(depend)s"' % vars()
339 for depend in rdepends:
340 print >> alldepends_file, '"%(package)s" -> "%(depend)s" [style=dashed]' % vars()
689 341
690 bb.debug(2, "Additional runtime dependencies for %s are: %s" % (item, " ".join(rdepends)))
691 342
692 for rdepend in rdepends: 343 # Add depends now
693 if rdepend in self.status.ignored_dependencies: 344 depends_file = file('depends.dot', 'w' )
694 continue 345 print >> depends_file, "digraph depends {"
695 if not self.buildRProvider(rdepend, buildAllDeps): 346 add_depends( pkgs_to_build )
696 return False 347 print >> depends_file, "}"
697 return True 348
349 # Add all depends now
350 alldepends_file = file('alldepends.dot', 'w' )
351 print >> alldepends_file, "digraph alldepends {"
352 add_all_depends( pkgs_to_build, [] )
353 print >> alldepends_file, "}"
698 354
699 def buildDepgraph( self ): 355 def buildDepgraph( self ):
700 all_depends = self.status.all_depends 356 all_depends = self.status.all_depends
@@ -702,6 +358,7 @@ class BBCooker:
702 358
703 localdata = data.createCopy(self.configuration.data) 359 localdata = data.createCopy(self.configuration.data)
704 bb.data.update_data(localdata) 360 bb.data.update_data(localdata)
361 bb.data.expandKeys(localdata)
705 362
706 def calc_bbfile_priority(filename): 363 def calc_bbfile_priority(filename):
707 for (regex, pri) in self.status.bbfile_config_priorities: 364 for (regex, pri) in self.status.bbfile_config_priorities:
@@ -712,9 +369,9 @@ class BBCooker:
712 # Handle PREFERRED_PROVIDERS 369 # Handle PREFERRED_PROVIDERS
713 for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split(): 370 for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
714 (providee, provider) = p.split(':') 371 (providee, provider) = p.split(':')
715 if providee in self.preferred and self.preferred[providee] != provider: 372 if providee in self.status.preferred and self.status.preferred[providee] != provider:
716 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee])) 373 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
717 self.preferred[providee] = provider 374 self.status.preferred[providee] = provider
718 375
719 # Calculate priorities for each file 376 # Calculate priorities for each file
720 for p in self.status.pkg_fn.keys(): 377 for p in self.status.pkg_fn.keys():
@@ -726,19 +383,19 @@ class BBCooker:
726 """ 383 """
727 all_depends = self.status.all_depends 384 all_depends = self.status.all_depends
728 pn_provides = self.status.pn_provides 385 pn_provides = self.status.pn_provides
729 bb.debug(1, "collating packages for \"world\"") 386 bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
730 for f in self.status.possible_world: 387 for f in self.status.possible_world:
731 terminal = True 388 terminal = True
732 pn = self.status.pkg_fn[f] 389 pn = self.status.pkg_fn[f]
733 390
734 for p in pn_provides[pn]: 391 for p in pn_provides[pn]:
735 if p.startswith('virtual/'): 392 if p.startswith('virtual/'):
736 bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p)) 393 bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
737 terminal = False 394 terminal = False
738 break 395 break
739 for pf in self.status.providers[p]: 396 for pf in self.status.providers[p]:
740 if self.status.pkg_fn[pf] != pn: 397 if self.status.pkg_fn[pf] != pn:
741 bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p)) 398 bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
742 terminal = False 399 terminal = False
743 break 400 break
744 if terminal: 401 if terminal:
@@ -748,13 +405,8 @@ class BBCooker:
748 self.status.possible_world = None 405 self.status.possible_world = None
749 self.status.all_depends = None 406 self.status.all_depends = None
750 407
751 def myProgressCallback( self, x, y, f, bb_cache, from_cache ): 408 def myProgressCallback( self, x, y, f, from_cache ):
752 # feed the status with new input 409 """Update any tty with the progress change"""
753
754 self.status.handle_bb_data(f, bb_cache, from_cache)
755
756 if bbdebug > 0:
757 return
758 if os.isatty(sys.stdout.fileno()): 410 if os.isatty(sys.stdout.fileno()):
759 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) ) 411 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
760 sys.stdout.flush() 412 sys.stdout.flush()
@@ -771,9 +423,10 @@ class BBCooker:
771 try: 423 try:
772 from bb import shell 424 from bb import shell
773 except ImportError, details: 425 except ImportError, details:
774 bb.fatal("Sorry, shell not available (%s)" % details ) 426 bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
775 else: 427 else:
776 bb.data.update_data( self.configuration.data ) 428 bb.data.update_data( self.configuration.data )
429 bb.data.expandKeys(localdata)
777 shell.start( self ) 430 shell.start( self )
778 sys.exit( 0 ) 431 sys.exit( 0 )
779 432
@@ -796,9 +449,9 @@ class BBCooker:
796 bb.event.register(var,bb.data.getVar(var, data)) 449 bb.event.register(var,bb.data.getVar(var, data))
797 450
798 except IOError: 451 except IOError:
799 bb.fatal( "Unable to open %s" % afile ) 452 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % afile )
800 except bb.parse.ParseError, details: 453 except bb.parse.ParseError, details:
801 bb.fatal( "Unable to parse %s (%s)" % (afile, details) ) 454 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
802 455
803 def handleCollections( self, collections ): 456 def handleCollections( self, collections ):
804 """Handle collections""" 457 """Handle collections"""
@@ -807,22 +460,22 @@ class BBCooker:
807 for c in collection_list: 460 for c in collection_list:
808 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1) 461 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
809 if regex == None: 462 if regex == None:
810 bb.error("BBFILE_PATTERN_%s not defined" % c) 463 bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
811 continue 464 continue
812 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1) 465 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
813 if priority == None: 466 if priority == None:
814 bb.error("BBFILE_PRIORITY_%s not defined" % c) 467 bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
815 continue 468 continue
816 try: 469 try:
817 cre = re.compile(regex) 470 cre = re.compile(regex)
818 except re.error: 471 except re.error:
819 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex)) 472 bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
820 continue 473 continue
821 try: 474 try:
822 pri = int(priority) 475 pri = int(priority)
823 self.status.bbfile_config_priorities.append((cre, pri)) 476 self.status.bbfile_config_priorities.append((cre, pri))
824 except ValueError: 477 except ValueError:
825 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority)) 478 bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
826 479
827 480
828 def cook( self, configuration, args ): 481 def cook( self, configuration, args ):
@@ -834,11 +487,16 @@ class BBCooker:
834 487
835 self.configuration = configuration 488 self.configuration = configuration
836 489
837 if not self.configuration.cmd: 490 if self.configuration.verbose:
838 self.configuration.cmd = "build" 491 bb.msg.set_verbose(True)
839 492
840 if self.configuration.debug: 493 if self.configuration.debug:
841 bb.debug_level = self.configuration.debug 494 bb.msg.set_debug_level(self.configuration.debug)
495 else:
496 bb.msg.set_debug_level(0)
497
498 if self.configuration.debug_domains:
499 bb.msg.set_debug_domains(self.configuration.debug_domains)
842 500
843 self.configuration.data = bb.data.init() 501 self.configuration.data = bb.data.init()
844 502
@@ -847,6 +505,12 @@ class BBCooker:
847 505
848 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) ) 506 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
849 507
508 if not self.configuration.cmd:
509 self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data)
510
511 # For backwards compatibility - REMOVE ME
512 if not self.configuration.cmd:
513 self.configuration.cmd = "build"
850 514
851 # 515 #
852 # Special updated configuration we use for firing events 516 # Special updated configuration we use for firing events
@@ -871,20 +535,34 @@ class BBCooker:
871 if self.configuration.buildfile is not None: 535 if self.configuration.buildfile is not None:
872 bf = os.path.abspath( self.configuration.buildfile ) 536 bf = os.path.abspath( self.configuration.buildfile )
873 try: 537 try:
874 bbfile_data = bb.parse.handle(bf, self.configuration.data) 538 os.stat(bf)
875 except IOError: 539 except OSError:
876 bb.fatal("Unable to open %s" % bf) 540 (filelist, masked) = self.collect_bbfiles()
541 regexp = re.compile(self.configuration.buildfile)
542 matches = []
543 for f in filelist:
544 if regexp.search(f) and os.path.isfile(f):
545 bf = f
546 matches.append(f)
547 if len(matches) != 1:
548 bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (self.configuration.buildfile, len(matches)))
549 for f in matches:
550 bb.msg.error(bb.msg.domain.Parsing, " %s" % f)
551 sys.exit(1)
552 bf = matches[0]
553
554 bbfile_data = bb.parse.handle(bf, self.configuration.data)
877 555
878 item = bb.data.getVar('PN', bbfile_data, 1) 556 item = bb.data.getVar('PN', bbfile_data, 1)
879 try: 557 try:
880 self.tryBuildPackage( bf, item, bbfile_data ) 558 self.tryBuildPackage(bf, item, self.configuration.cmd, bbfile_data, True)
881 except bb.build.EventException: 559 except bb.build.EventException:
882 bb.error( "Build of '%s' failed" % item ) 560 bb.msg.error(bb.msg.domain.Build, "Build of '%s' failed" % item )
883 561
884 sys.exit( self.stats.show() ) 562 sys.exit( self.stats.show() )
885 563
886 # initialise the parsing status now we know we will need deps 564 # initialise the parsing status now we know we will need deps
887 self.status = BBParsingStatus() 565 self.status = bb.cache.CacheData()
888 566
889 ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or "" 567 ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
890 self.status.ignored_dependencies = Set( ignore.split() ) 568 self.status.ignored_dependencies = Set( ignore.split() )
@@ -912,23 +590,23 @@ class BBCooker:
912 try: 590 try:
913 import psyco 591 import psyco
914 except ImportError: 592 except ImportError:
915 if bbdebug == 0: 593 bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
916 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
917 else: 594 else:
918 psyco.bind( self.collect_bbfiles ) 595 psyco.bind( self.parse_bbfiles )
919 else: 596 else:
920 bb.note("You have disabled Psyco. This decreases performance.") 597 bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
921 598
922 try: 599 try:
923 bb.debug(1, "collecting .bb files") 600 bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
924 self.collect_bbfiles( self.myProgressCallback ) 601 (filelist, masked) = self.collect_bbfiles()
925 bb.debug(1, "parsing complete") 602 self.parse_bbfiles(filelist, masked, self.myProgressCallback)
926 if bbdebug == 0: 603 bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
927 print 604 print
928 if self.configuration.parse_only: 605 if self.configuration.parse_only:
929 print "Requested parsing .bb files only. Exiting." 606 bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only. Exiting.")
930 return 607 return
931 608
609
932 self.buildDepgraph() 610 self.buildDepgraph()
933 611
934 if self.configuration.show_versions: 612 if self.configuration.show_versions:
@@ -940,30 +618,41 @@ class BBCooker:
940 for t in self.status.world_target: 618 for t in self.status.world_target:
941 pkgs_to_build.append(t) 619 pkgs_to_build.append(t)
942 620
621 if self.configuration.dot_graph:
622 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
623 sys.exit( 0 )
624
943 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.event_data)) 625 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.event_data))
944 626
945 failures = 0 627 localdata = data.createCopy(self.configuration.data)
946 for k in pkgs_to_build: 628 bb.data.update_data(localdata)
947 failed = False 629 bb.data.expandKeys(localdata)
948 try: 630
949 if self.buildProvider( k , False ) == 0: 631 taskdata = bb.taskdata.TaskData(self.configuration.abort)
950 # already diagnosed 632
951 failed = True 633 runlist = []
952 except bb.build.EventException: 634 try:
953 bb.error("Build of " + k + " failed") 635 for k in pkgs_to_build:
954 failed = True 636 taskdata.add_provider(localdata, self.status, k)
955 637 runlist.append([k, "do_%s" % self.configuration.cmd])
956 if failed: 638 taskdata.add_unresolved(localdata, self.status)
957 failures += failures 639 except bb.providers.NoProvider:
958 if self.configuration.abort: 640 sys.exit(1)
959 sys.exit(1) 641
642 rq = bb.runqueue.RunQueue()
643 rq.prepare_runqueue(self.configuration.data, self.status, taskdata, runlist)
644 try:
645 failures = rq.execute_runqueue(self, self.configuration.data, self.status, taskdata, runlist)
646 except runqueue.TaskFailure, (fnid, fn, taskname):
647 bb.msg.error(bb.msg.domain.Build, "'%s, %s' failed" % (fn, taskname))
648 sys.exit(1)
960 649
961 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.event_data, failures)) 650 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.event_data, failures))
962 651
963 sys.exit( self.stats.show() ) 652 sys.exit( self.stats.show() )
964 653
965 except KeyboardInterrupt: 654 except KeyboardInterrupt:
966 print "\nNOTE: KeyboardInterrupt - Build not completed." 655 bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
967 sys.exit(1) 656 sys.exit(1)
968 657
969 def get_bbfiles( self, path = os.getcwd() ): 658 def get_bbfiles( self, path = os.getcwd() ):
@@ -985,9 +674,8 @@ class BBCooker:
985 return [] 674 return []
986 return finddata.readlines() 675 return finddata.readlines()
987 676
988 def collect_bbfiles( self, progressCallback ): 677 def collect_bbfiles( self ):
989 """Collect all available .bb build files""" 678 """Collect all available .bb build files"""
990 self.cb = progressCallback
991 parsed, cached, skipped, masked = 0, 0, 0, 0 679 parsed, cached, skipped, masked = 0, 0, 0, 0
992 self.bb_cache = bb.cache.init(self) 680 self.bb_cache = bb.cache.init(self)
993 681
@@ -998,7 +686,7 @@ class BBCooker:
998 files = self.get_bbfiles() 686 files = self.get_bbfiles()
999 687
1000 if not len(files): 688 if not len(files):
1001 bb.error("no files to build.") 689 bb.msg.error(bb.msg.domain.Collection, "no files to build.")
1002 690
1003 newfiles = [] 691 newfiles = []
1004 for f in files: 692 for f in files:
@@ -1009,62 +697,80 @@ class BBCooker:
1009 continue 697 continue
1010 newfiles += glob.glob(f) or [ f ] 698 newfiles += glob.glob(f) or [ f ]
1011 699
1012 bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1) or "" 700 bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
701
702 if not bbmask:
703 return (newfiles, 0)
704
1013 try: 705 try:
1014 bbmask_compiled = re.compile(bbmask) 706 bbmask_compiled = re.compile(bbmask)
1015 except sre_constants.error: 707 except sre_constants.error:
1016 bb.fatal("BBMASK is not a valid regular expression.") 708 bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
1017 709
710 finalfiles = []
1018 for i in xrange( len( newfiles ) ): 711 for i in xrange( len( newfiles ) ):
1019 f = newfiles[i] 712 f = newfiles[i]
1020 if bbmask and bbmask_compiled.search(f): 713 if bbmask and bbmask_compiled.search(f):
1021 bb.debug(1, "bbmake: skipping %s" % f) 714 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
1022 masked += 1 715 masked += 1
1023 continue 716 continue
1024 debug(1, "bbmake: parsing %s" % f) 717 finalfiles.append(f)
718
719 return (finalfiles, masked)
720
721 def parse_bbfiles(self, filelist, masked, progressCallback = None):
722 parsed, cached, skipped = 0, 0, 0
723 for i in xrange( len( filelist ) ):
724 f = filelist[i]
725
726 bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
1025 727
1026 # read a file's metadata 728 # read a file's metadata
1027 try: 729 try:
1028 fromCache, skip = self.bb_cache.loadData(f, self) 730 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data)
1029 if skip: 731 if skip:
1030 skipped += 1 732 skipped += 1
1031 #bb.note("Skipping %s" % f) 733 bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
1032 self.bb_cache.skip(f) 734 self.bb_cache.skip(f)
1033 continue 735 continue
1034 elif fromCache: cached += 1 736 elif fromCache: cached += 1
1035 else: parsed += 1 737 else: parsed += 1
1036 deps = None 738 deps = None
1037 739
740 # Disabled by RP as was no longer functional
1038 # allow metadata files to add items to BBFILES 741 # allow metadata files to add items to BBFILES
1039 #data.update_data(self.pkgdata[f]) 742 #data.update_data(self.pkgdata[f])
1040 addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None 743 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
1041 if addbbfiles: 744 #if addbbfiles:
1042 for aof in addbbfiles.split(): 745 # for aof in addbbfiles.split():
1043 if not files.count(aof): 746 # if not files.count(aof):
1044 if not os.path.isabs(aof): 747 # if not os.path.isabs(aof):
1045 aof = os.path.join(os.path.dirname(f),aof) 748 # aof = os.path.join(os.path.dirname(f),aof)
1046 files.append(aof) 749 # files.append(aof)
750
751 self.bb_cache.handle_data(f, self.status)
1047 752
1048 # now inform the caller 753 # now inform the caller
1049 if self.cb is not None: 754 if progressCallback is not None:
1050 self.cb( i + 1, len( newfiles ), f, self.bb_cache, fromCache ) 755 progressCallback( i + 1, len( filelist ), f, fromCache )
1051 756
1052 except IOError, e: 757 except IOError, e:
1053 self.bb_cache.remove(f) 758 self.bb_cache.remove(f)
1054 bb.error("opening %s: %s" % (f, e)) 759 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
1055 pass 760 pass
1056 except KeyboardInterrupt: 761 except KeyboardInterrupt:
1057 self.bb_cache.sync() 762 self.bb_cache.sync()
1058 raise 763 raise
1059 except Exception, e: 764 except Exception, e:
1060 self.bb_cache.remove(f) 765 self.bb_cache.remove(f)
1061 bb.error("%s while parsing %s" % (e, f)) 766 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
1062 except: 767 except:
1063 self.bb_cache.remove(f) 768 self.bb_cache.remove(f)
1064 raise 769 raise
1065 770
1066 if self.cb is not None: 771 if progressCallback is not None:
1067 print "\rNOTE: Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ), 772 print "\r" # need newline after Handling Bitbake files message
773 bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
1068 774
1069 self.bb_cache.sync() 775 self.bb_cache.sync()
1070 776
@@ -1090,11 +796,11 @@ Default BBFILES are the .bb files in the current directory.""" )
1090 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status", 796 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
1091 action = "store_true", dest = "force", default = False ) 797 action = "store_true", dest = "force", default = False )
1092 798
1093 parser.add_option( "-i", "--interactive", help = "drop into the interactive mode.", 799 parser.add_option( "-i", "--interactive", help = "drop into the interactive mode also called the BitBake shell.",
1094 action = "store_true", dest = "interactive", default = False ) 800 action = "store_true", dest = "interactive", default = False )
1095 801
1096 parser.add_option( "-c", "--cmd", help = "Specify task to execute. Note that this only executes the specified task for the providee and the packages it depends on, i.e. 'compile' does not implicitly call stage for the dependencies (IOW: use only if you know what you are doing)", 802 parser.add_option( "-c", "--cmd", help = "Specify task to execute. Note that this only executes the specified task for the providee and the packages it depends on, i.e. 'compile' does not implicitly call stage for the dependencies (IOW: use only if you know what you are doing). Depending on the base.bbclass a listtaks tasks is defined and will show available tasks",
1097 action = "store", dest = "cmd", default = "build" ) 803 action = "store", dest = "cmd" )
1098 804
1099 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf", 805 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
1100 action = "append", dest = "file", default = [] ) 806 action = "append", dest = "file", default = [] )
@@ -1102,7 +808,7 @@ Default BBFILES are the .bb files in the current directory.""" )
1102 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal", 808 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
1103 action = "store_true", dest = "verbose", default = False ) 809 action = "store_true", dest = "verbose", default = False )
1104 810
1105 parser.add_option( "-D", "--debug", help = "Increase the debug level", 811 parser.add_option( "-D", "--debug", help = "Increase the debug level. You can specify this more than once.",
1106 action = "count", dest="debug", default = 0) 812 action = "count", dest="debug", default = 0)
1107 813
1108 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions", 814 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
@@ -1120,6 +826,16 @@ Default BBFILES are the .bb files in the current directory.""" )
1120 parser.add_option( "-e", "--environment", help = "show the global or per-package environment (this is what used to be bbread)", 826 parser.add_option( "-e", "--environment", help = "show the global or per-package environment (this is what used to be bbread)",
1121 action = "store_true", dest = "show_environment", default = False ) 827 action = "store_true", dest = "show_environment", default = False )
1122 828
829 parser.add_option( "-g", "--graphviz", help = "emit the dependency trees of the specified packages in the dot syntax",
830 action = "store_true", dest = "dot_graph", default = False )
831
832 parser.add_option( "-I", "--ignore-deps", help = """Stop processing at the given list of dependencies when generating dependency graphs. This can help to make the graph more appealing""",
833 action = "append", dest = "ignored_dot_deps", default = [] )
834
835 parser.add_option( "-l", "--log-domains", help = """Show debug logging for the specified logging domains""",
836 action = "append", dest = "debug_domains", default = [] )
837
838
1123 options, args = parser.parse_args( sys.argv ) 839 options, args = parser.parse_args( sys.argv )
1124 840
1125 cooker = BBCooker() 841 cooker = BBCooker()
@@ -1129,3 +845,9 @@ Default BBFILES are the .bb files in the current directory.""" )
1129 845
1130if __name__ == "__main__": 846if __name__ == "__main__":
1131 main() 847 main()
848 sys.exit(0)
849 import profile
850 profile.run('main()', "profile.log")
851 import pstats
852 p = pstats.Stats('profile.log')
853 p.print_stats()
diff --git a/bitbake/bin/bitdoc b/bitbake/bin/bitdoc
index 84d2ee2..e865e1b 100755
--- a/bitbake/bin/bitdoc
+++ b/bitbake/bin/bitdoc
@@ -442,7 +442,7 @@ Create a set of html pages (documentation) for a bitbake.conf....
442 options, args = parser.parse_args( sys.argv ) 442 options, args = parser.parse_args( sys.argv )
443 443
444 if options.debug: 444 if options.debug:
445 bb.debug_level = options.debug 445 bb.msg.set_debug_level(options.debug)
446 446
447 return options.config, options.output 447 return options.config, options.output
448 448
diff --git a/bitbake/classes/base.bbclass b/bitbake/classes/base.bbclass
index 1d75964..cfb82a4 100644
--- a/bitbake/classes/base.bbclass
+++ b/bitbake/classes/base.bbclass
@@ -41,7 +41,7 @@ bbdebug() {
41 exit 1 41 exit 1
42 } 42 }
43 43
44 test ${@bb.debug_level} -ge $1 && { 44 test ${@bb.msg.debug_level} -ge $1 && {
45 shift 45 shift
46 echo "DEBUG:" $* 46 echo "DEBUG:" $*
47 } 47 }
diff --git a/bitbake/conf/bitbake.conf b/bitbake/conf/bitbake.conf
index d288fee..19a3fe8 100644
--- a/bitbake/conf/bitbake.conf
+++ b/bitbake/conf/bitbake.conf
@@ -26,11 +26,12 @@ DEPLOY_DIR_IMAGE = "${DEPLOY_DIR}/images"
26DL_DIR = "${TMPDIR}/downloads" 26DL_DIR = "${TMPDIR}/downloads"
27FETCHCOMMAND = "" 27FETCHCOMMAND = ""
28FETCHCOMMAND_cvs = "/usr/bin/env cvs -d${CVSROOT} co ${CVSCOOPTS} ${CVSMODULE}" 28FETCHCOMMAND_cvs = "/usr/bin/env cvs -d${CVSROOT} co ${CVSCOOPTS} ${CVSMODULE}"
29FETCHCOMMAND_svn = "/usr/bin/env svn co http://${SVNROOT} ${SVNCOOPTS} ${SVNMODULE}" 29FETCHCOMMAND_svn = "/usr/bin/env svn co ${SVNCOOPTS} ${SVNROOT} ${SVNMODULE}"
30FETCHCOMMAND_wget = "/usr/bin/env wget -t 5 --passive-ftp -P ${DL_DIR} ${URI}" 30FETCHCOMMAND_wget = "/usr/bin/env wget -t 5 --passive-ftp -P ${DL_DIR} ${URI}"
31FILESDIR = "${@bb.which(bb.data.getVar('FILESPATH', d, 1), '.')}" 31FILESDIR = "${@bb.which(bb.data.getVar('FILESPATH', d, 1), '.')}"
32FILESPATH = "${FILE_DIRNAME}/${PF}:${FILE_DIRNAME}/${P}:${FILE_DIRNAME}/${PN}:${FILE_DIRNAME}/files:${FILE_DIRNAME}" 32FILESPATH = "${FILE_DIRNAME}/${PF}:${FILE_DIRNAME}/${P}:${FILE_DIRNAME}/${PN}:${FILE_DIRNAME}/files:${FILE_DIRNAME}"
33FILE_DIRNAME = "${@os.path.dirname(bb.data.getVar('FILE', d))}" 33FILE_DIRNAME = "${@os.path.dirname(bb.data.getVar('FILE', d))}"
34GITDIR = "${DL_DIR}/git"
34IMAGE_CMD = "_NO_DEFINED_IMAGE_TYPES_" 35IMAGE_CMD = "_NO_DEFINED_IMAGE_TYPES_"
35IMAGE_ROOTFS = "${TMPDIR}/rootfs" 36IMAGE_ROOTFS = "${TMPDIR}/rootfs"
36MKTEMPCMD = "mktemp -q ${TMPBASE}" 37MKTEMPCMD = "mktemp -q ${TMPBASE}"
@@ -47,9 +48,11 @@ RESUMECOMMAND_wget = "/usr/bin/env wget -c -t 5 --passive-ftp -P ${DL_DIR} ${URI
47S = "${WORKDIR}/${P}" 48S = "${WORKDIR}/${P}"
48SRC_URI = "file://${FILE}" 49SRC_URI = "file://${FILE}"
49STAMP = "${TMPDIR}/stamps/${PF}" 50STAMP = "${TMPDIR}/stamps/${PF}"
51SVNDIR = "${DL_DIR}/svn"
50T = "${WORKDIR}/temp" 52T = "${WORKDIR}/temp"
51TARGET_ARCH = "${BUILD_ARCH}" 53TARGET_ARCH = "${BUILD_ARCH}"
52TMPDIR = "${TOPDIR}/tmp" 54TMPDIR = "${TOPDIR}/tmp"
53UPDATECOMMAND = "" 55UPDATECOMMAND = ""
54UPDATECOMMAND_cvs = "/usr/bin/env cvs -d${CVSROOT} update ${CVSCOOPTS}" 56UPDATECOMMAND_cvs = "/usr/bin/env cvs -d${CVSROOT} update ${CVSCOOPTS}"
57UPDATECOMMAND_svn = "/usr/bin/env svn update ${SVNCOOPTS}"
55WORKDIR = "${TMPDIR}/work/${PF}" 58WORKDIR = "${TMPDIR}/work/${PF}"
diff --git a/bitbake/contrib/vim/ftdetect/bitbake.vim b/bitbake/contrib/vim/ftdetect/bitbake.vim
new file mode 100644
index 0000000..3882a9a
--- /dev/null
+++ b/bitbake/contrib/vim/ftdetect/bitbake.vim
@@ -0,0 +1,4 @@
1au BufNewFile,BufRead *.bb setfiletype bitbake
2au BufNewFile,BufRead *.bbclass setfiletype bitbake
3au BufNewFile,BufRead *.inc setfiletype bitbake
4" au BufNewFile,BufRead *.conf setfiletype bitbake
diff --git a/bitbake/contrib/vim/syntax/bitbake.vim b/bitbake/contrib/vim/syntax/bitbake.vim
index 5d2bc63..43a1990 100644
--- a/bitbake/contrib/vim/syntax/bitbake.vim
+++ b/bitbake/contrib/vim/syntax/bitbake.vim
@@ -42,11 +42,11 @@ syn region bbString matchgroup=bbQuote start=/'/ skip=/\\$/ excludenl end=/'/ c
42 42
43syn keyword bbExportFlag export contained nextgroup=bbIdentifier skipwhite 43syn keyword bbExportFlag export contained nextgroup=bbIdentifier skipwhite
44syn match bbVarDeref "${[a-zA-Z0-9\-_\.]\+}" contained 44syn match bbVarDeref "${[a-zA-Z0-9\-_\.]\+}" contained
45syn match bbVarDef "^\(export\s*\)\?\([a-zA-Z0-9\-_\.]\+\(_[${}a-zA-Z0-9\-_\.]\+\)\?\)\s*\(\(:=\)\|\(+=\)\|\(=+\)\|\(?=\)\|=\)\@=" contains=bbExportFlag,bbIdentifier,bbVarDeref nextgroup=bbVarEq 45syn match bbVarDef "^\(export\s*\)\?\([a-zA-Z0-9\-_\.]\+\(_[${}a-zA-Z0-9\-_\.]\+\)\?\)\s*\(:=\|+=\|=+\|\.=\|=\.\|?=\|=\)\@=" contains=bbExportFlag,bbIdentifier,bbVarDeref nextgroup=bbVarEq
46 46
47syn match bbIdentifier "[a-zA-Z0-9\-_\.]\+" display contained 47syn match bbIdentifier "[a-zA-Z0-9\-_\.]\+" display contained
48"syn keyword bbVarEq = display contained nextgroup=bbVarValue 48"syn keyword bbVarEq = display contained nextgroup=bbVarValue
49syn match bbVarEq "\(:=\)\|\(+=\)\|\(=+\)\|\(?=\)\|=" contained nextgroup=bbVarValue 49syn match bbVarEq "\(:=\|+=\|=+\|\.=\|=\.\|?=\|=\)" contained nextgroup=bbVarValue
50syn match bbVarValue ".*$" contained contains=bbString,bbVarDeref 50syn match bbVarValue ".*$" contained contains=bbString,bbVarDeref
51 51
52 52
@@ -90,8 +90,8 @@ syn region bbDefRegion start='^def\s\+\w\+\s*([^)]*)\s*:\s*$' end='^\(\s\|$\)\@
90 90
91 91
92" BitBake statements 92" BitBake statements
93syn keyword bbStatement include inherit addtask addhandler EXPORT_FUNCTIONS display contained 93syn keyword bbStatement include inherit require addtask addhandler EXPORT_FUNCTIONS display contained
94syn match bbStatementLine "^\(include\|inherit\|addtask\|addhandler\|EXPORT_FUNCTIONS\)\s\+" contains=bbStatement nextgroup=bbStatementRest 94syn match bbStatementLine "^\(include\|inherit\|require\|addtask\|addhandler\|EXPORT_FUNCTIONS\)\s\+" contains=bbStatement nextgroup=bbStatementRest
95syn match bbStatementRest ".*$" contained contains=bbString,bbVarDeref 95syn match bbStatementRest ".*$" contained contains=bbString,bbVarDeref
96 96
97" Highlight 97" Highlight
diff --git a/bitbake/doc/manual/usermanual.xml b/bitbake/doc/manual/usermanual.xml
index c314236..7eb1203 100644
--- a/bitbake/doc/manual/usermanual.xml
+++ b/bitbake/doc/manual/usermanual.xml
@@ -17,7 +17,7 @@
17 <holder>Phil Blundell</holder> 17 <holder>Phil Blundell</holder>
18 </copyright> 18 </copyright>
19 <legalnotice> 19 <legalnotice>
20 <para>This work is licensed under the Creative Commons Attribution License. To view a copy of this license, visit <ulink url="http://creativecommons.org/licenses/by/2.0/">http://creativecommons.org/licenses/by/2.0/</ulink> or send a letter to Creative Commons, 559 Nathan Abbott Way, Stanford, California 94305, USA.</para> 20 <para>This work is licensed under the Creative Commons Attribution License. To view a copy of this license, visit <ulink url="http://creativecommons.org/licenses/by/2.5/">http://creativecommons.org/licenses/by/2.5/</ulink> or send a letter to Creative Commons, 559 Nathan Abbott Way, Stanford, California 94305, USA.</para>
21 </legalnotice> 21 </legalnotice>
22 </bookinfo> 22 </bookinfo>
23 <chapter> 23 <chapter>
@@ -195,7 +195,7 @@ addtask printdate before do_build</screen></para>
195 <section> 195 <section>
196 <title>Events</title> 196 <title>Events</title>
197 <para><emphasis>NOTE:</emphasis> This is only supported in .bb and .bbclass files.</para> 197 <para><emphasis>NOTE:</emphasis> This is only supported in .bb and .bbclass files.</para>
198 <para>BitBake also implements a means of registering event handlers. Events are triggered at certain points during operation, such as, the beginning of operation against a given .bb, the start of a given task, task failure, task success, et cetera. The intent was to make it easy to do things like email notifications on build failure.</para> 198 <para>BitBake allows to install event handlers. Events are triggered at certain points during operation, such as, the beginning of operation against a given .bb, the start of a given task, task failure, task success, et cetera. The intent was to make it easy to do things like email notifications on build failure.</para>
199 <para><screen>addhandler myclass_eventhandler 199 <para><screen>addhandler myclass_eventhandler
200python myclass_eventhandler() { 200python myclass_eventhandler() {
201 from bb.event import NotHandled, getName 201 from bb.event import NotHandled, getName
@@ -205,6 +205,7 @@ python myclass_eventhandler() {
205 print "The file we run for is %s" % data.getVar('FILE', e.data, True) 205 print "The file we run for is %s" % data.getVar('FILE', e.data, True)
206 206
207 return NotHandled 207 return NotHandled
208}
208</screen></para><para> 209</screen></para><para>
209This event handler gets called every time an event is triggered. A global variable <varname>e</varname> is defined. <varname>e</varname>.data contains an instance of bb.data. With the getName(<varname>e</varname>) 210This event handler gets called every time an event is triggered. A global variable <varname>e</varname> is defined. <varname>e</varname>.data contains an instance of bb.data. With the getName(<varname>e</varname>)
210method one can get the name of the triggered event.</para><para>The above event handler prints the name 211method one can get the name of the triggered event.</para><para>The above event handler prints the name
@@ -344,15 +345,19 @@ options:
344 cannot be remade, the other dependencies of these 345 cannot be remade, the other dependencies of these
345 targets can be processed all the same. 346 targets can be processed all the same.
346 -f, --force force run of specified cmd, regardless of stamp status 347 -f, --force force run of specified cmd, regardless of stamp status
347 -i, --interactive drop into the interactive mode. 348 -i, --interactive drop into the interactive mode also called the BitBake
349 shell.
348 -c CMD, --cmd=CMD Specify task to execute. Note that this only executes 350 -c CMD, --cmd=CMD Specify task to execute. Note that this only executes
349 the specified task for the providee and the packages 351 the specified task for the providee and the packages
350 it depends on, i.e. 'compile' does not implicitly call 352 it depends on, i.e. 'compile' does not implicitly call
351 stage for the dependencies (IOW: use only if you know 353 stage for the dependencies (IOW: use only if you know
352 what you are doing) 354 what you are doing). Depending on the base.bbclass a
355 listtaks tasks is defined and will show available
356 tasks
353 -r FILE, --read=FILE read the specified file before bitbake.conf 357 -r FILE, --read=FILE read the specified file before bitbake.conf
354 -v, --verbose output more chit-chat to the terminal 358 -v, --verbose output more chit-chat to the terminal
355 -D, --debug Increase the debug level 359 -D, --debug Increase the debug level. You can specify this more
360 than once.
356 -n, --dry-run don't execute, just go through the motions 361 -n, --dry-run don't execute, just go through the motions
357 -p, --parse-only quit after parsing the BB files (developers only) 362 -p, --parse-only quit after parsing the BB files (developers only)
358 -d, --disable-psyco disable using the psyco just-in-time compiler (not 363 -d, --disable-psyco disable using the psyco just-in-time compiler (not
@@ -360,6 +365,12 @@ options:
360 -s, --show-versions show current and preferred versions of all packages 365 -s, --show-versions show current and preferred versions of all packages
361 -e, --environment show the global or per-package environment (this is 366 -e, --environment show the global or per-package environment (this is
362 what used to be bbread) 367 what used to be bbread)
368 -g, --graphviz emit the dependency trees of the specified packages in
369 the dot syntax
370 -I IGNORED_DOT_DEPS, --ignore-deps=IGNORED_DOT_DEPS
371 Stop processing at the given list of dependencies when
372 generating dependency graphs. This can help to make
373 the graph more appealing
363 374
364</screen> 375</screen>
365 </para> 376 </para>
@@ -386,6 +397,14 @@ options:
386 <screen><prompt>$ </prompt>bitbake virtual/whatever</screen> 397 <screen><prompt>$ </prompt>bitbake virtual/whatever</screen>
387 <screen><prompt>$ </prompt>bitbake -c clean virtual/whatever</screen> 398 <screen><prompt>$ </prompt>bitbake -c clean virtual/whatever</screen>
388 </example> 399 </example>
400 <example>
401 <title>Generating dependency graphs</title>
402 <para>BitBake is able to generate dependency graphs using the dot syntax. These graphs can be converted
403to images using the <application>dot</application> application from <ulink url="http://www.graphviz.org">graphviz</ulink>.
404Three files will be written into the current working directory, <emphasis>depends.dot</emphasis> containing <varname>DEPENDS</varname> variables, <emphasis>rdepends.dot</emphasis> and <emphasis>alldepends.dot</emphasis> containing both <varname>DEPENDS</varname> and <varname>RDEPENDS</varname>. To stop depending on common depends one can use the <prompt>-I depend</prompt> to omit these from the graph. This can lead to more readable graphs. E.g. this way <varname>DEPENDS</varname> from inherited classes, e.g. base.bbclass, can be removed from the graph.</para>
405 <screen><prompt>$ </prompt>bitbake -g blah</screen>
406 <screen><prompt>$ </prompt>bitbake -g -I virtual/whatever -I bloom blah</screen>
407 </example>
389 </para> 408 </para>
390 </section> 409 </section>
391 <section> 410 <section>
diff --git a/bitbake/lib/bb/COW.py b/bitbake/lib/bb/COW.py
new file mode 100644
index 0000000..826d435
--- /dev/null
+++ b/bitbake/lib/bb/COW.py
@@ -0,0 +1,305 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4This is a copy on write dictionary and set which abuses classes to try and be nice and fast.
5
6Please Note:
7 Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW.
8 Assign a file to __warn__ to get warnings about slow operations.
9"""
10
11from inspect import getmro
12
13import copy
14import types, sets
15types.ImmutableTypes = tuple([ \
16 types.BooleanType, \
17 types.ComplexType, \
18 types.FloatType, \
19 types.IntType, \
20 types.LongType, \
21 types.NoneType, \
22 types.TupleType, \
23 sets.ImmutableSet] + \
24 list(types.StringTypes))
25
26MUTABLE = "__mutable__"
27
28class COWMeta(type):
29 pass
30
31class COWDictMeta(COWMeta):
32 __warn__ = False
33 __hasmutable__ = False
34 __marker__ = tuple()
35
36 def __str__(cls):
37 # FIXME: I have magic numbers!
38 return "<COWDict Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) - 3)
39 __repr__ = __str__
40
41 def cow(cls):
42 class C(cls):
43 __count__ = cls.__count__ + 1
44 return C
45 copy = cow
46 __call__ = cow
47
48 def __setitem__(cls, key, value):
49 if not isinstance(value, types.ImmutableTypes):
50 if not isinstance(value, COWMeta):
51 cls.__hasmutable__ = True
52 key += MUTABLE
53 setattr(cls, key, value)
54
55 def __getmutable__(cls, key, readonly=False):
56 nkey = key + MUTABLE
57 try:
58 return cls.__dict__[nkey]
59 except KeyError:
60 pass
61
62 value = getattr(cls, nkey)
63 if readonly:
64 return value
65
66 if not cls.__warn__ is False and not isinstance(value, COWMeta):
67 print >> cls.__warn__, "Warning: Doing a copy because %s is a mutable type." % key
68 try:
69 value = value.copy()
70 except AttributeError, e:
71 value = copy.copy(value)
72 setattr(cls, nkey, value)
73 return value
74
75 __getmarker__ = []
76 def __getreadonly__(cls, key, default=__getmarker__):
77 """\
78 Get a value (even if mutable) which you promise not to change.
79 """
80 return cls.__getitem__(key, default, True)
81
82 def __getitem__(cls, key, default=__getmarker__, readonly=False):
83 try:
84 try:
85 value = getattr(cls, key)
86 except AttributeError:
87 value = cls.__getmutable__(key, readonly)
88
89 # This is for values which have been deleted
90 if value is cls.__marker__:
91 raise AttributeError("key %s does not exist." % key)
92
93 return value
94 except AttributeError, e:
95 if not default is cls.__getmarker__:
96 return default
97
98 raise KeyError(str(e))
99
100 def __delitem__(cls, key):
101 cls.__setitem__(key, cls.__marker__)
102
103 def __revertitem__(cls, key):
104 if not cls.__dict__.has_key(key):
105 key += MUTABLE
106 delattr(cls, key)
107
108 def has_key(cls, key):
109 value = cls.__getreadonly__(key, cls.__marker__)
110 if value is cls.__marker__:
111 return False
112 return True
113
114 def iter(cls, type, readonly=False):
115 for key in dir(cls):
116 if key.startswith("__"):
117 continue
118
119 if key.endswith(MUTABLE):
120 key = key[:-len(MUTABLE)]
121
122 if type == "keys":
123 yield key
124
125 try:
126 if readonly:
127 value = cls.__getreadonly__(key)
128 else:
129 value = cls[key]
130 except KeyError:
131 continue
132
133 if type == "values":
134 yield value
135 if type == "items":
136 yield (key, value)
137 raise StopIteration()
138
139 def iterkeys(cls):
140 return cls.iter("keys")
141 def itervalues(cls, readonly=False):
142 if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False:
143 print >> cls.__warn__, "Warning: If you arn't going to change any of the values call with True."
144 return cls.iter("values", readonly)
145 def iteritems(cls, readonly=False):
146 if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False:
147 print >> cls.__warn__, "Warning: If you arn't going to change any of the values call with True."
148 return cls.iter("items", readonly)
149
150class COWSetMeta(COWDictMeta):
151 def __str__(cls):
152 # FIXME: I have magic numbers!
153 return "<COWSet Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) -3)
154 __repr__ = __str__
155
156 def cow(cls):
157 class C(cls):
158 __count__ = cls.__count__ + 1
159 return C
160
161 def add(cls, value):
162 COWDictMeta.__setitem__(cls, repr(hash(value)), value)
163
164 def remove(cls, value):
165 COWDictMeta.__delitem__(cls, repr(hash(value)))
166
167 def __in__(cls, value):
168 return COWDictMeta.has_key(repr(hash(value)))
169
170 def iterkeys(cls):
171 raise TypeError("sets don't have keys")
172
173 def iteritems(cls):
174 raise TypeError("sets don't have 'items'")
175
176# These are the actual classes you use!
177class COWDictBase(object):
178 __metaclass__ = COWDictMeta
179 __count__ = 0
180
181class COWSetBase(object):
182 __metaclass__ = COWSetMeta
183 __count__ = 0
184
185if __name__ == "__main__":
186 import sys
187 COWDictBase.__warn__ = sys.stderr
188 a = COWDictBase()
189 print "a", a
190
191 a['a'] = 'a'
192 a['b'] = 'b'
193 a['dict'] = {}
194
195 b = a.copy()
196 print "b", b
197 b['c'] = 'b'
198
199 print
200
201 print "a", a
202 for x in a.iteritems():
203 print x
204 print "--"
205 print "b", b
206 for x in b.iteritems():
207 print x
208 print
209
210 b['dict']['a'] = 'b'
211 b['a'] = 'c'
212
213 print "a", a
214 for x in a.iteritems():
215 print x
216 print "--"
217 print "b", b
218 for x in b.iteritems():
219 print x
220 print
221
222 try:
223 b['dict2']
224 except KeyError, e:
225 print "Okay!"
226
227 a['set'] = COWSetBase()
228 a['set'].add("o1")
229 a['set'].add("o1")
230 a['set'].add("o2")
231
232 print "a", a
233 for x in a['set'].itervalues():
234 print x
235 print "--"
236 print "b", b
237 for x in b['set'].itervalues():
238 print x
239 print
240
241 b['set'].add('o3')
242
243 print "a", a
244 for x in a['set'].itervalues():
245 print x
246 print "--"
247 print "b", b
248 for x in b['set'].itervalues():
249 print x
250 print
251
252 a['set2'] = set()
253 a['set2'].add("o1")
254 a['set2'].add("o1")
255 a['set2'].add("o2")
256
257 print "a", a
258 for x in a.iteritems():
259 print x
260 print "--"
261 print "b", b
262 for x in b.iteritems(readonly=True):
263 print x
264 print
265
266 del b['b']
267 try:
268 print b['b']
269 except KeyError:
270 print "Yay! deleted key raises error"
271
272 if b.has_key('b'):
273 print "Boo!"
274 else:
275 print "Yay - has_key with delete works!"
276
277 print "a", a
278 for x in a.iteritems():
279 print x
280 print "--"
281 print "b", b
282 for x in b.iteritems(readonly=True):
283 print x
284 print
285
286 b.__revertitem__('b')
287
288 print "a", a
289 for x in a.iteritems():
290 print x
291 print "--"
292 print "b", b
293 for x in b.iteritems(readonly=True):
294 print x
295 print
296
297 b.__revertitem__('dict')
298 print "a", a
299 for x in a.iteritems():
300 print x
301 print "--"
302 print "b", b
303 for x in b.iteritems(readonly=True):
304 print x
305 print
diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py
index c3e7a16..61eb5f3 100644
--- a/bitbake/lib/bb/__init__.py
+++ b/bitbake/lib/bb/__init__.py
@@ -23,7 +23,7 @@ this program; if not, write to the Free Software Foundation, Inc., 59 Temple
23Place, Suite 330, Boston, MA 02111-1307 USA. 23Place, Suite 330, Boston, MA 02111-1307 USA.
24""" 24"""
25 25
26__version__ = "1.4.3" 26__version__ = "1.7.4"
27 27
28__all__ = [ 28__all__ = [
29 29
@@ -63,24 +63,24 @@ __all__ = [
63 "manifest", 63 "manifest",
64 "methodpool", 64 "methodpool",
65 "cache", 65 "cache",
66 "runqueue",
67 "taskdata",
68 "providers",
66 ] 69 ]
67 70
68whitespace = '\t\n\x0b\x0c\r ' 71whitespace = '\t\n\x0b\x0c\r '
69lowercase = 'abcdefghijklmnopqrstuvwxyz' 72lowercase = 'abcdefghijklmnopqrstuvwxyz'
70 73
71import sys, os, types, re, string 74import sys, os, types, re, string, bb
75from bb import msg
72 76
73#projectdir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) 77#projectdir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
74projectdir = os.getcwd() 78projectdir = os.getcwd()
75 79
76debug_level = 0
77
78if "BBDEBUG" in os.environ: 80if "BBDEBUG" in os.environ:
79 level = int(os.environ["BBDEBUG"]) 81 level = int(os.environ["BBDEBUG"])
80 if level: 82 if level:
81 debug_level = level 83 bb.msg.set_debug_level(level)
82 else:
83 debug_level = 0
84 84
85class VarExpandError(Exception): 85class VarExpandError(Exception):
86 pass 86 pass
@@ -99,22 +99,17 @@ class MalformedUrl(Exception):
99####################################################################### 99#######################################################################
100####################################################################### 100#######################################################################
101 101
102debug_prepend = ''
103
104
105def debug(lvl, *args): 102def debug(lvl, *args):
106 if debug_level >= lvl: 103 bb.msg.std_debug(lvl, ''.join(args))
107 print debug_prepend + 'DEBUG:', ''.join(args)
108 104
109def note(*args): 105def note(*args):
110 print debug_prepend + 'NOTE:', ''.join(args) 106 bb.msg.std_note(''.join(args))
111 107
112def error(*args): 108def error(*args):
113 print debug_prepend + 'ERROR:', ''.join(args) 109 bb.msg.std_error(''.join(args))
114 110
115def fatal(*args): 111def fatal(*args):
116 print debug_prepend + 'ERROR:', ''.join(args) 112 bb.msg.std_fatal(''.join(args))
117 sys.exit(1)
118 113
119 114
120####################################################################### 115#######################################################################
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index 8e169e0..942bdc1 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -25,18 +25,9 @@ You should have received a copy of the GNU General Public License along with
25Based on functions from the base bb module, Copyright 2003 Holger Schurig 25Based on functions from the base bb module, Copyright 2003 Holger Schurig
26""" 26"""
27 27
28from bb import debug, data, fetch, fatal, error, note, event, mkdirhier, utils 28from bb import data, fetch, event, mkdirhier, utils
29import bb, os 29import bb, os
30 30
31# data holds flags and function name for a given task
32_task_data = data.init()
33
34# graph represents task interdependencies
35_task_graph = bb.digraph()
36
37# stack represents execution order, excepting dependencies
38_task_stack = []
39
40# events 31# events
41class FuncFailed(Exception): 32class FuncFailed(Exception):
42 """Executed function failed""" 33 """Executed function failed"""
@@ -76,13 +67,6 @@ class InvalidTask(TaskBase):
76 67
77# functions 68# functions
78 69
79def init(data):
80 global _task_data, _task_graph, _task_stack
81 _task_data = data.init()
82 _task_graph = bb.digraph()
83 _task_stack = []
84
85
86def exec_func(func, d, dirs = None): 70def exec_func(func, d, dirs = None):
87 """Execute an BB 'function'""" 71 """Execute an BB 'function'"""
88 72
@@ -163,7 +147,7 @@ def exec_func_shell(func, d):
163 147
164 f = open(runfile, "w") 148 f = open(runfile, "w")
165 f.write("#!/bin/sh -e\n") 149 f.write("#!/bin/sh -e\n")
166 if bb.debug_level > 0: f.write("set -x\n") 150 if bb.msg.debug_level['default'] > 0: f.write("set -x\n")
167 data.emit_env(f, d) 151 data.emit_env(f, d)
168 152
169 f.write("cd %s\n" % os.getcwd()) 153 f.write("cd %s\n" % os.getcwd())
@@ -171,18 +155,18 @@ def exec_func_shell(func, d):
171 f.close() 155 f.close()
172 os.chmod(runfile, 0775) 156 os.chmod(runfile, 0775)
173 if not func: 157 if not func:
174 error("Function not specified") 158 bb.msg.error(bb.msg.domain.Build, "Function not specified")
175 raise FuncFailed() 159 raise FuncFailed()
176 160
177 # open logs 161 # open logs
178 si = file('/dev/null', 'r') 162 si = file('/dev/null', 'r')
179 try: 163 try:
180 if bb.debug_level > 0: 164 if bb.msg.debug_level['default'] > 0:
181 so = os.popen("tee \"%s\"" % logfile, "w") 165 so = os.popen("tee \"%s\"" % logfile, "w")
182 else: 166 else:
183 so = file(logfile, 'w') 167 so = file(logfile, 'w')
184 except OSError, e: 168 except OSError, e:
185 bb.error("opening log file: %s" % e) 169 bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e)
186 pass 170 pass
187 171
188 se = so 172 se = so
@@ -205,7 +189,10 @@ def exec_func_shell(func, d):
205 else: 189 else:
206 maybe_fakeroot = '' 190 maybe_fakeroot = ''
207 ret = os.system('%ssh -e %s' % (maybe_fakeroot, runfile)) 191 ret = os.system('%ssh -e %s' % (maybe_fakeroot, runfile))
208 os.chdir(prevdir) 192 try:
193 os.chdir(prevdir)
194 except:
195 pass
209 196
210 if not interact: 197 if not interact:
211 # restore the backups 198 # restore the backups
@@ -224,14 +211,14 @@ def exec_func_shell(func, d):
224 os.close(ose[0]) 211 os.close(ose[0])
225 212
226 if ret==0: 213 if ret==0:
227 if bb.debug_level > 0: 214 if bb.msg.debug_level['default'] > 0:
228 os.remove(runfile) 215 os.remove(runfile)
229# os.remove(logfile) 216# os.remove(logfile)
230 return 217 return
231 else: 218 else:
232 error("function %s failed" % func) 219 bb.msg.error(bb.msg.domain.Build, "function %s failed" % func)
233 if data.getVar("BBINCLUDELOGS", d): 220 if data.getVar("BBINCLUDELOGS", d):
234 error("log data follows (%s)" % logfile) 221 bb.msg.error(bb.msg.domain.Build, "log data follows (%s)" % logfile)
235 f = open(logfile, "r") 222 f = open(logfile, "r")
236 while True: 223 while True:
237 l = f.readline() 224 l = f.readline()
@@ -241,7 +228,7 @@ def exec_func_shell(func, d):
241 print '| %s' % l 228 print '| %s' % l
242 f.close() 229 f.close()
243 else: 230 else:
244 error("see log in %s" % logfile) 231 bb.msg.error(bb.msg.domain.Build, "see log in %s" % logfile)
245 raise FuncFailed( logfile ) 232 raise FuncFailed( logfile )
246 233
247 234
@@ -281,7 +268,7 @@ def exec_task(task, d):
281 return 1 268 return 1
282 269
283 try: 270 try:
284 debug(1, "Executing task %s" % item) 271 bb.msg.debug(1, bb.msg.domain.Build, "Executing task %s" % item)
285 old_overrides = data.getVar('OVERRIDES', d, 0) 272 old_overrides = data.getVar('OVERRIDES', d, 0)
286 localdata = data.createCopy(d) 273 localdata = data.createCopy(d)
287 data.setVar('OVERRIDES', 'task_%s:%s' % (item, old_overrides), localdata) 274 data.setVar('OVERRIDES', 'task_%s:%s' % (item, old_overrides), localdata)
@@ -292,21 +279,63 @@ def exec_task(task, d):
292 task_cache.append(item) 279 task_cache.append(item)
293 data.setVar('_task_cache', task_cache, d) 280 data.setVar('_task_cache', task_cache, d)
294 except FuncFailed, reason: 281 except FuncFailed, reason:
295 note( "Task failed: %s" % reason ) 282 bb.msg.note(1, bb.msg.domain.Build, "Task failed: %s" % reason )
296 failedevent = TaskFailed(item, d) 283 failedevent = TaskFailed(item, d)
297 event.fire(failedevent) 284 event.fire(failedevent)
298 raise EventException("Function failed in task: %s" % reason, failedevent) 285 raise EventException("Function failed in task: %s" % reason, failedevent)
299 286
300 # execute 287 if data.getVarFlag(task, 'dontrundeps', d):
301 task_graph.walkdown(task, execute) 288 execute(None, task)
289 else:
290 task_graph.walkdown(task, execute)
302 291
303 # make stamp, or cause event and raise exception 292 # make stamp, or cause event and raise exception
304 if not data.getVarFlag(task, 'nostamp', d): 293 if not data.getVarFlag(task, 'nostamp', d):
305 mkstamp(task, d) 294 mkstamp(task, d)
306 295
296def stamp_is_current_cache(dataCache, file_name, task, checkdeps = 1):
297 """
298 Check status of a given task's stamp.
299 Returns 0 if it is not current and needs updating.
300 Same as stamp_is_current but works against the dataCache instead of d
301 """
302 task_graph = dataCache.task_queues[file_name]
303
304 if not dataCache.stamp[file_name]:
305 return 0
306
307 stampfile = "%s.%s" % (dataCache.stamp[file_name], task)
308 if not os.access(stampfile, os.F_OK):
309 return 0
310
311 if checkdeps == 0:
312 return 1
313
314 import stat
315 tasktime = os.stat(stampfile)[stat.ST_MTIME]
316
317 _deps = []
318 def checkStamp(graph, task):
319 # check for existance
320 if 'nostamp' in dataCache.task_deps[file_name] and task in dataCache.task_deps[file_name]['nostamp']:
321 return 1
322
323 if not stamp_is_current_cache(dataCache, file_name, task, 0):
324 return 0
325
326 depfile = "%s.%s" % (dataCache.stamp[file_name], task)
327 deptime = os.stat(depfile)[stat.ST_MTIME]
328 if deptime > tasktime:
329 return 0
330 return 1
331
332 return task_graph.walkdown(task, checkStamp)
307 333
308def stamp_is_current(task, d, checkdeps = 1): 334def stamp_is_current(task, d, checkdeps = 1):
309 """Check status of a given task's stamp. returns 0 if it is not current and needs updating.""" 335 """
336 Check status of a given task's stamp.
337 Returns 0 if it is not current and needs updating.
338 """
310 task_graph = data.getVar('_task_graph', d) 339 task_graph = data.getVar('_task_graph', d)
311 if not task_graph: 340 if not task_graph:
312 task_graph = bb.digraph() 341 task_graph = bb.digraph()
@@ -360,7 +389,6 @@ def mkstamp(task, d):
360 f = open(stamp, "w") 389 f = open(stamp, "w")
361 f.close() 390 f.close()
362 391
363
364def add_task(task, deps, d): 392def add_task(task, deps, d):
365 task_graph = data.getVar('_task_graph', d) 393 task_graph = data.getVar('_task_graph', d)
366 if not task_graph: 394 if not task_graph:
@@ -374,6 +402,21 @@ def add_task(task, deps, d):
374 # don't assume holding a reference 402 # don't assume holding a reference
375 data.setVar('_task_graph', task_graph, d) 403 data.setVar('_task_graph', task_graph, d)
376 404
405 task_deps = data.getVar('_task_deps', d)
406 if not task_deps:
407 task_deps = {}
408 def getTask(name):
409 deptask = data.getVarFlag(task, name, d)
410 if deptask:
411 if not name in task_deps:
412 task_deps[name] = {}
413 task_deps[name][task] = deptask
414 getTask('deptask')
415 getTask('rdeptask')
416 getTask('recrdeptask')
417 getTask('nostamp')
418
419 data.setVar('_task_deps', task_deps, d)
377 420
378def remove_task(task, kill, d): 421def remove_task(task, kill, d):
379 """Remove an BB 'task'. 422 """Remove an BB 'task'.
@@ -399,6 +442,3 @@ def task_exists(task, d):
399 task_graph = bb.digraph() 442 task_graph = bb.digraph()
400 data.setVar('_task_graph', task_graph, d) 443 data.setVar('_task_graph', task_graph, d)
401 return task_graph.hasnode(task) 444 return task_graph.hasnode(task)
402
403def get_task_data():
404 return _task_data
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index 921a9f7..05c4251 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -33,15 +33,15 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
33import os, re 33import os, re
34import bb.data 34import bb.data
35import bb.utils 35import bb.utils
36from sets import Set
36 37
37try: 38try:
38 import cPickle as pickle 39 import cPickle as pickle
39except ImportError: 40except ImportError:
40 import pickle 41 import pickle
41 print "NOTE: Importing cPickle failed. Falling back to a very slow implementation." 42 bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
42 43
43# __cache_version__ = "123" 44__cache_version__ = "125"
44__cache_version__ = "124" # changes the __depends structure
45 45
46class Cache: 46class Cache:
47 """ 47 """
@@ -58,14 +58,12 @@ class Cache:
58 58
59 if self.cachedir in [None, '']: 59 if self.cachedir in [None, '']:
60 self.has_cache = False 60 self.has_cache = False
61 if cooker.cb is not None: 61 bb.msg.note(1, bb.msg.domain.Cache, "Not using a cache. Set CACHE = <directory> to enable.")
62 print "NOTE: Not using a cache. Set CACHE = <directory> to enable."
63 else: 62 else:
64 self.has_cache = True 63 self.has_cache = True
65 self.cachefile = os.path.join(self.cachedir,"bb_cache.dat") 64 self.cachefile = os.path.join(self.cachedir,"bb_cache.dat")
66 65
67 if cooker.cb is not None: 66 bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s'" % self.cachedir)
68 print "NOTE: Using cache in '%s'" % self.cachedir
69 try: 67 try:
70 os.stat( self.cachedir ) 68 os.stat( self.cachedir )
71 except OSError: 69 except OSError:
@@ -80,7 +78,7 @@ class Cache:
80 if version_data['BITBAKE_VER'] != bb.__version__: 78 if version_data['BITBAKE_VER'] != bb.__version__:
81 raise ValueError, 'Bitbake Version Mismatch' 79 raise ValueError, 'Bitbake Version Mismatch'
82 except (ValueError, KeyError): 80 except (ValueError, KeyError):
83 bb.note("Invalid cache found, rebuilding...") 81 bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...")
84 self.depends_cache = {} 82 self.depends_cache = {}
85 83
86 if self.depends_cache: 84 if self.depends_cache:
@@ -108,7 +106,7 @@ class Cache:
108 if fn != self.data_fn: 106 if fn != self.data_fn:
109 # We're trying to access data in the cache which doesn't exist 107 # We're trying to access data in the cache which doesn't exist
110 # yet setData hasn't been called to setup the right access. Very bad. 108 # yet setData hasn't been called to setup the right access. Very bad.
111 bb.error("Parsing error data_fn %s and fn %s don't match" % (self.data_fn, fn)) 109 bb.msg.error(bb.msg.domain.Cache, "Parsing error data_fn %s and fn %s don't match" % (self.data_fn, fn))
112 110
113 result = bb.data.getVar(var, self.data, exp) 111 result = bb.data.getVar(var, self.data, exp)
114 self.depends_cache[fn][var] = result 112 self.depends_cache[fn][var] = result
@@ -127,15 +125,15 @@ class Cache:
127 self.getVar("__depends", fn, True) 125 self.getVar("__depends", fn, True)
128 self.depends_cache[fn]["CACHETIMESTAMP"] = bb.parse.cached_mtime(fn) 126 self.depends_cache[fn]["CACHETIMESTAMP"] = bb.parse.cached_mtime(fn)
129 127
130 def loadDataFull(self, fn, cooker): 128 def loadDataFull(self, fn, cfgData):
131 """ 129 """
132 Return a complete set of data for fn. 130 Return a complete set of data for fn.
133 To do this, we need to parse the file. 131 To do this, we need to parse the file.
134 """ 132 """
135 bb_data, skipped = self.load_bbfile(fn, cooker) 133 bb_data, skipped = self.load_bbfile(fn, cfgData)
136 return bb_data 134 return bb_data
137 135
138 def loadData(self, fn, cooker): 136 def loadData(self, fn, cfgData):
139 """ 137 """
140 Load a subset of data for fn. 138 Load a subset of data for fn.
141 If the cached data is valid we do nothing, 139 If the cached data is valid we do nothing,
@@ -148,7 +146,7 @@ class Cache:
148 return True, True 146 return True, True
149 return True, False 147 return True, False
150 148
151 bb_data, skipped = self.load_bbfile(fn, cooker) 149 bb_data, skipped = self.load_bbfile(fn, cfgData)
152 self.setData(fn, bb_data) 150 self.setData(fn, bb_data)
153 return False, skipped 151 return False, skipped
154 152
@@ -175,32 +173,36 @@ class Cache:
175 173
176 # Check file still exists 174 # Check file still exists
177 if self.mtime(fn) == 0: 175 if self.mtime(fn) == 0:
178 bb.debug(2, "Cache: %s not longer exists" % fn) 176 bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s not longer exists" % fn)
179 self.remove(fn) 177 self.remove(fn)
180 return False 178 return False
181 179
182 # File isn't in depends_cache 180 # File isn't in depends_cache
183 if not fn in self.depends_cache: 181 if not fn in self.depends_cache:
184 bb.debug(2, "Cache: %s is not cached" % fn) 182 bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s is not cached" % fn)
185 self.remove(fn) 183 self.remove(fn)
186 return False 184 return False
187 185
188 # Check the file's timestamp 186 # Check the file's timestamp
189 if bb.parse.cached_mtime(fn) > self.getVar("CACHETIMESTAMP", fn, True): 187 if bb.parse.cached_mtime(fn) > self.getVar("CACHETIMESTAMP", fn, True):
190 bb.debug(2, "Cache: %s changed" % fn) 188 bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s changed" % fn)
191 self.remove(fn) 189 self.remove(fn)
192 return False 190 return False
193 191
194 # Check dependencies are still valid 192 # Check dependencies are still valid
195 depends = self.getVar("__depends", fn, True) 193 depends = self.getVar("__depends", fn, True)
196 for f,old_mtime in depends: 194 for f,old_mtime in depends:
195 # Check if file still exists
196 if self.mtime(f) == 0:
197 return False
198
197 new_mtime = bb.parse.cached_mtime(f) 199 new_mtime = bb.parse.cached_mtime(f)
198 if (new_mtime > old_mtime): 200 if (new_mtime > old_mtime):
199 bb.debug(2, "Cache: %s's dependency %s changed" % (fn, f)) 201 bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s's dependency %s changed" % (fn, f))
200 self.remove(fn) 202 self.remove(fn)
201 return False 203 return False
202 204
203 bb.debug(2, "Depends Cache: %s is clean" % fn) 205 bb.msg.debug(2, bb.msg.domain.Cache, "Depends Cache: %s is clean" % fn)
204 if not fn in self.clean: 206 if not fn in self.clean:
205 self.clean[fn] = "" 207 self.clean[fn] = ""
206 208
@@ -220,7 +222,7 @@ class Cache:
220 Remove a fn from the cache 222 Remove a fn from the cache
221 Called from the parser in error cases 223 Called from the parser in error cases
222 """ 224 """
223 bb.debug(1, "Removing %s from cache" % fn) 225 bb.msg.debug(1, bb.msg.domain.Cache, "Removing %s from cache" % fn)
224 if fn in self.depends_cache: 226 if fn in self.depends_cache:
225 del self.depends_cache[fn] 227 del self.depends_cache[fn]
226 if fn in self.clean: 228 if fn in self.clean:
@@ -229,7 +231,7 @@ class Cache:
229 def sync(self): 231 def sync(self):
230 """ 232 """
231 Save the cache 233 Save the cache
232 Called from the parser when complete (or exitting) 234 Called from the parser when complete (or exiting)
233 """ 235 """
234 236
235 if not self.has_cache: 237 if not self.has_cache:
@@ -243,12 +245,103 @@ class Cache:
243 p.dump([self.depends_cache, version_data]) 245 p.dump([self.depends_cache, version_data])
244 246
245 def mtime(self, cachefile): 247 def mtime(self, cachefile):
246 try: 248 return bb.parse.cached_mtime_noerror(cachefile)
247 return os.stat(cachefile)[8]
248 except OSError:
249 return 0
250 249
251 def load_bbfile( self, bbfile , cooker): 250 def handle_data(self, file_name, cacheData):
251 """
252 Save data we need into the cache
253 """
254
255 pn = self.getVar('PN', file_name, True)
256 pv = self.getVar('PV', file_name, True)
257 pr = self.getVar('PR', file_name, True)
258 dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
259 provides = Set([pn] + (self.getVar("PROVIDES", file_name, True) or "").split())
260 depends = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "")
261 packages = (self.getVar('PACKAGES', file_name, True) or "").split()
262 packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
263 rprovides = (self.getVar("RPROVIDES", file_name, True) or "").split()
264
265 cacheData.task_queues[file_name] = self.getVar("_task_graph", file_name, True)
266 cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name, True)
267
268 # build PackageName to FileName lookup table
269 if pn not in cacheData.pkg_pn:
270 cacheData.pkg_pn[pn] = []
271 cacheData.pkg_pn[pn].append(file_name)
272
273 cacheData.stamp[file_name] = self.getVar('STAMP', file_name, True)
274
275 # build FileName to PackageName lookup table
276 cacheData.pkg_fn[file_name] = pn
277 cacheData.pkg_pvpr[file_name] = (pv,pr)
278 cacheData.pkg_dp[file_name] = dp
279
280 # Build forward and reverse provider hashes
281 # Forward: virtual -> [filenames]
282 # Reverse: PN -> [virtuals]
283 if pn not in cacheData.pn_provides:
284 cacheData.pn_provides[pn] = Set()
285 cacheData.pn_provides[pn] |= provides
286
287 for provide in provides:
288 if provide not in cacheData.providers:
289 cacheData.providers[provide] = []
290 cacheData.providers[provide].append(file_name)
291
292 cacheData.deps[file_name] = Set()
293 for dep in depends:
294 cacheData.all_depends.add(dep)
295 cacheData.deps[file_name].add(dep)
296
297 # Build reverse hash for PACKAGES, so runtime dependencies
298 # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
299 for package in packages:
300 if not package in cacheData.packages:
301 cacheData.packages[package] = []
302 cacheData.packages[package].append(file_name)
303 rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split()
304
305 for package in packages_dynamic:
306 if not package in cacheData.packages_dynamic:
307 cacheData.packages_dynamic[package] = []
308 cacheData.packages_dynamic[package].append(file_name)
309
310 for rprovide in rprovides:
311 if not rprovide in cacheData.rproviders:
312 cacheData.rproviders[rprovide] = []
313 cacheData.rproviders[rprovide].append(file_name)
314
315 # Build hash of runtime depends and rececommends
316
317 def add_dep(deplist, deps):
318 for dep in deps:
319 if not dep in deplist:
320 deplist[dep] = ""
321
322 if not file_name in cacheData.rundeps:
323 cacheData.rundeps[file_name] = {}
324 if not file_name in cacheData.runrecs:
325 cacheData.runrecs[file_name] = {}
326
327 for package in packages + [pn]:
328 if not package in cacheData.rundeps[file_name]:
329 cacheData.rundeps[file_name][package] = {}
330 if not package in cacheData.runrecs[file_name]:
331 cacheData.runrecs[file_name][package] = {}
332
333 add_dep(cacheData.rundeps[file_name][package], bb.utils.explode_deps(self.getVar('RDEPENDS', file_name, True) or ""))
334 add_dep(cacheData.runrecs[file_name][package], bb.utils.explode_deps(self.getVar('RRECOMMENDS', file_name, True) or ""))
335 add_dep(cacheData.rundeps[file_name][package], bb.utils.explode_deps(self.getVar("RDEPENDS_%s" % package, file_name, True) or ""))
336 add_dep(cacheData.runrecs[file_name][package], bb.utils.explode_deps(self.getVar("RRECOMMENDS_%s" % package, file_name, True) or ""))
337
338 # Collect files we may need for possible world-dep
339 # calculations
340 if not self.getVar('BROKEN', file_name, True) and not self.getVar('EXCLUDE_FROM_WORLD', file_name, True):
341 cacheData.possible_world.append(file_name)
342
343
344 def load_bbfile( self, bbfile , config):
252 """ 345 """
253 Load and parse one .bb build file 346 Load and parse one .bb build file
254 Return the data and whether parsing resulted in the file being skipped 347 Return the data and whether parsing resulted in the file being skipped
@@ -257,25 +350,15 @@ class Cache:
257 import bb 350 import bb
258 from bb import utils, data, parse, debug, event, fatal 351 from bb import utils, data, parse, debug, event, fatal
259 352
260 topdir = data.getVar('TOPDIR', cooker.configuration.data)
261 if not topdir:
262 topdir = os.path.abspath(os.getcwd())
263 # set topdir to here
264 data.setVar('TOPDIR', topdir, cooker.configuration)
265 bbfile = os.path.abspath(bbfile)
266 bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
267 # expand tmpdir to include this topdir 353 # expand tmpdir to include this topdir
268 data.setVar('TMPDIR', data.getVar('TMPDIR', cooker.configuration.data, 1) or "", cooker.configuration.data) 354 data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config)
269 # set topdir to location of .bb file 355 bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
270 topdir = bbfile_loc
271 #data.setVar('TOPDIR', topdir, cfg)
272 # go there
273 oldpath = os.path.abspath(os.getcwd()) 356 oldpath = os.path.abspath(os.getcwd())
274 if self.mtime(topdir): 357 if self.mtime(bbfile_loc):
275 os.chdir(topdir) 358 os.chdir(bbfile_loc)
276 bb_data = data.init_db(cooker.configuration.data) 359 bb_data = data.init_db(config)
277 try: 360 try:
278 parse.handle(bbfile, bb_data) # read .bb data 361 bb_data = parse.handle(bbfile, bb_data) # read .bb data
279 os.chdir(oldpath) 362 os.chdir(oldpath)
280 return bb_data, False 363 return bb_data, False
281 except bb.parse.SkipPackage: 364 except bb.parse.SkipPackage:
@@ -304,3 +387,45 @@ def init(cooker):
304 """ 387 """
305 return Cache(cooker) 388 return Cache(cooker)
306 389
390
391
392#============================================================================#
393# CacheData
394#============================================================================#
395class CacheData:
396 """
397 The data structures we compile from the cached data
398 """
399
400 def __init__(self):
401 """
402 Direct cache variables
403 (from Cache.handle_data)
404 """
405 self.providers = {}
406 self.rproviders = {}
407 self.packages = {}
408 self.packages_dynamic = {}
409 self.possible_world = []
410 self.pkg_pn = {}
411 self.pkg_fn = {}
412 self.pkg_pvpr = {}
413 self.pkg_dp = {}
414 self.pn_provides = {}
415 self.all_depends = Set()
416 self.deps = {}
417 self.rundeps = {}
418 self.runrecs = {}
419 self.task_queues = {}
420 self.task_deps = {}
421 self.stamp = {}
422 self.preferred = {}
423
424 """
425 Indirect Cache variables
426 (set elsewhere)
427 """
428 self.ignored_dependencies = []
429 self.world_target = Set()
430 self.bbfile_priority = {}
431 self.bbfile_config_priorities = []
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py
index 55d1cc9..819dff9 100644
--- a/bitbake/lib/bb/data.py
+++ b/bitbake/lib/bb/data.py
@@ -45,7 +45,8 @@ else:
45 path = os.path.dirname(os.path.dirname(sys.argv[0])) 45 path = os.path.dirname(os.path.dirname(sys.argv[0]))
46sys.path.insert(0,path) 46sys.path.insert(0,path)
47 47
48from bb import note, debug, data_smart 48from bb import data_smart
49import bb
49 50
50_dict_type = data_smart.DataSmart 51_dict_type = data_smart.DataSmart
51 52
@@ -362,10 +363,12 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
362 val.rstrip() 363 val.rstrip()
363 if not val: 364 if not val:
364 return 0 365 return 0
366
367 varExpanded = expand(var, d)
365 368
366 if getVarFlag(var, "func", d): 369 if getVarFlag(var, "func", d):
367# NOTE: should probably check for unbalanced {} within the var 370# NOTE: should probably check for unbalanced {} within the var
368 o.write("%s() {\n%s\n}\n" % (var, val)) 371 o.write("%s() {\n%s\n}\n" % (varExpanded, val))
369 else: 372 else:
370 if getVarFlag(var, "export", d): 373 if getVarFlag(var, "export", d):
371 o.write('export ') 374 o.write('export ')
@@ -375,7 +378,7 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
375# if we're going to output this within doublequotes, 378# if we're going to output this within doublequotes,
376# to a shell, we need to escape the quotes in the var 379# to a shell, we need to escape the quotes in the var
377 alter = re.sub('"', '\\"', val.strip()) 380 alter = re.sub('"', '\\"', val.strip())
378 o.write('%s="%s"\n' % (var, alter)) 381 o.write('%s="%s"\n' % (varExpanded, alter))
379 return 1 382 return 1
380 383
381 384
@@ -430,8 +433,38 @@ def update_data(d):
430 >>> update_data(d) 433 >>> update_data(d)
431 >>> print getVar('TEST', d) 434 >>> print getVar('TEST', d)
432 local 435 local
436
437 CopyMonster:
438 >>> e = d.createCopy()
439 >>> setVar('TEST_foo', 'foo', e)
440 >>> update_data(e)
441 >>> print getVar('TEST', e)
442 local
443
444 >>> setVar('OVERRIDES', 'arm:ramses:local:foo', e)
445 >>> update_data(e)
446 >>> print getVar('TEST', e)
447 foo
448
449 >>> f = d.createCopy()
450 >>> setVar('TEST_moo', 'something', f)
451 >>> setVar('OVERRIDES', 'moo:arm:ramses:local:foo', e)
452 >>> update_data(e)
453 >>> print getVar('TEST', e)
454 foo
455
456
457 >>> h = init()
458 >>> setVar('SRC_URI', 'file://append.foo;patch=1 ', h)
459 >>> g = h.createCopy()
460 >>> setVar('SRC_URI_append_arm', 'file://other.foo;patch=1', g)
461 >>> setVar('OVERRIDES', 'arm:moo', g)
462 >>> update_data(g)
463 >>> print getVar('SRC_URI', g)
464 file://append.foo;patch=1 file://other.foo;patch=1
465
433 """ 466 """
434 debug(2, "update_data()") 467 bb.msg.debug(2, bb.msg.domain.Data, "update_data()")
435 468
436 # now ask the cookie monster for help 469 # now ask the cookie monster for help
437 #print "Cookie Monster" 470 #print "Cookie Monster"
@@ -460,7 +493,7 @@ def update_data(d):
460 l = len(o)+1 493 l = len(o)+1
461 494
462 # see if one should even try 495 # see if one should even try
463 if not o in d._seen_overrides: 496 if not d._seen_overrides.has_key(o):
464 continue 497 continue
465 498
466 vars = d._seen_overrides[o] 499 vars = d._seen_overrides[o]
@@ -469,10 +502,10 @@ def update_data(d):
469 try: 502 try:
470 d[name] = d[var] 503 d[name] = d[var]
471 except: 504 except:
472 note ("Untracked delVar") 505 bb.msg.note(1, bb.msg.domain.Data, "Untracked delVar")
473 506
474 # now on to the appends and prepends 507 # now on to the appends and prepends
475 if '_append' in d._special_values: 508 if d._special_values.has_key('_append'):
476 appends = d._special_values['_append'] or [] 509 appends = d._special_values['_append'] or []
477 for append in appends: 510 for append in appends:
478 for (a, o) in getVarFlag(append, '_append', d) or []: 511 for (a, o) in getVarFlag(append, '_append', d) or []:
@@ -487,7 +520,7 @@ def update_data(d):
487 setVar(append, sval, d) 520 setVar(append, sval, d)
488 521
489 522
490 if '_prepend' in d._special_values: 523 if d._special_values.has_key('_prepend'):
491 prepends = d._special_values['_prepend'] or [] 524 prepends = d._special_values['_prepend'] or []
492 525
493 for prepend in prepends: 526 for prepend in prepends:
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py
index fbd4167..054b852 100644
--- a/bitbake/lib/bb/data_smart.py
+++ b/bitbake/lib/bb/data_smart.py
@@ -29,14 +29,12 @@ Based on functions from the base bb module, Copyright 2003 Holger Schurig
29""" 29"""
30 30
31import copy, os, re, sys, time, types 31import copy, os, re, sys, time, types
32from bb import note, debug, error, fatal, utils, methodpool 32import bb
33from bb import utils, methodpool
34from COW import COWDictBase
33from sets import Set 35from sets import Set
36from new import classobj
34 37
35try:
36 import cPickle as pickle
37except ImportError:
38 import pickle
39 print "NOTE: Importing cPickle failed. Falling back to a very slow implementation."
40 38
41__setvar_keyword__ = ["_append","_prepend"] 39__setvar_keyword__ = ["_append","_prepend"]
42__setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend)(_(?P<add>.*))?') 40__setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend)(_(?P<add>.*))?')
@@ -45,12 +43,14 @@ __expand_python_regexp__ = re.compile(r"\${@.+?}")
45 43
46 44
47class DataSmart: 45class DataSmart:
48 def __init__(self): 46 def __init__(self, special = COWDictBase.copy(), seen = COWDictBase.copy() ):
49 self.dict = {} 47 self.dict = {}
50 48
51 # cookie monster tribute 49 # cookie monster tribute
52 self._special_values = {} 50 self._special_values = special
53 self._seen_overrides = {} 51 self._seen_overrides = seen
52
53 self.expand_cache = {}
54 54
55 def expand(self,s, varname): 55 def expand(self,s, varname):
56 def var_sub(match): 56 def var_sub(match):
@@ -75,6 +75,9 @@ class DataSmart:
75 if type(s) is not types.StringType: # sanity check 75 if type(s) is not types.StringType: # sanity check
76 return s 76 return s
77 77
78 if varname and varname in self.expand_cache:
79 return self.expand_cache[varname]
80
78 while s.find('$') != -1: 81 while s.find('$') != -1:
79 olds = s 82 olds = s
80 try: 83 try:
@@ -82,15 +85,20 @@ class DataSmart:
82 s = __expand_python_regexp__.sub(python_sub, s) 85 s = __expand_python_regexp__.sub(python_sub, s)
83 if s == olds: break 86 if s == olds: break
84 if type(s) is not types.StringType: # sanity check 87 if type(s) is not types.StringType: # sanity check
85 error('expansion of %s returned non-string %s' % (olds, s)) 88 bb.msg.error(bb.msg.domain.Data, 'expansion of %s returned non-string %s' % (olds, s))
86 except KeyboardInterrupt: 89 except KeyboardInterrupt:
87 raise 90 raise
88 except: 91 except:
89 note("%s:%s while evaluating:\n%s" % (sys.exc_info()[0], sys.exc_info()[1], s)) 92 bb.msg.note(1, bb.msg.domain.Data, "%s:%s while evaluating:\n%s" % (sys.exc_info()[0], sys.exc_info()[1], s))
90 raise 93 raise
94
95 if varname:
96 self.expand_cache[varname] = s
97
91 return s 98 return s
92 99
93 def initVar(self, var): 100 def initVar(self, var):
101 self.expand_cache = {}
94 if not var in self.dict: 102 if not var in self.dict:
95 self.dict[var] = {} 103 self.dict[var] = {}
96 104
@@ -119,6 +127,7 @@ class DataSmart:
119 self.initVar(var) 127 self.initVar(var)
120 128
121 def setVar(self,var,value): 129 def setVar(self,var,value):
130 self.expand_cache = {}
122 match = __setvar_regexp__.match(var) 131 match = __setvar_regexp__.match(var)
123 if match and match.group("keyword") in __setvar_keyword__: 132 if match and match.group("keyword") in __setvar_keyword__:
124 base = match.group('base') 133 base = match.group('base')
@@ -128,6 +137,7 @@ class DataSmart:
128 l.append([value, override]) 137 l.append([value, override])
129 self.setVarFlag(base, keyword, l) 138 self.setVarFlag(base, keyword, l)
130 139
140 # todo make sure keyword is not __doc__ or __module__
131 # pay the cookie monster 141 # pay the cookie monster
132 try: 142 try:
133 self._special_values[keyword].add( base ) 143 self._special_values[keyword].add( base )
@@ -135,10 +145,6 @@ class DataSmart:
135 self._special_values[keyword] = Set() 145 self._special_values[keyword] = Set()
136 self._special_values[keyword].add( base ) 146 self._special_values[keyword].add( base )
137 147
138 # SRC_URI_append_simpad is both a flag and a override
139 #if not override in self._seen_overrides:
140 # self._seen_overrides[override] = Set()
141 #self._seen_overrides[override].add( base )
142 return 148 return
143 149
144 if not var in self.dict: 150 if not var in self.dict:
@@ -150,7 +156,7 @@ class DataSmart:
150 # more cookies for the cookie monster 156 # more cookies for the cookie monster
151 if '_' in var: 157 if '_' in var:
152 override = var[var.rfind('_')+1:] 158 override = var[var.rfind('_')+1:]
153 if not override in self._seen_overrides: 159 if not self._seen_overrides.has_key(override):
154 self._seen_overrides[override] = Set() 160 self._seen_overrides[override] = Set()
155 self._seen_overrides[override].add( var ) 161 self._seen_overrides[override].add( var )
156 162
@@ -165,6 +171,7 @@ class DataSmart:
165 return value 171 return value
166 172
167 def delVar(self,var): 173 def delVar(self,var):
174 self.expand_cache = {}
168 self.dict[var] = {} 175 self.dict[var] = {}
169 176
170 def setVarFlag(self,var,flag,flagvalue): 177 def setVarFlag(self,var,flag,flagvalue):
@@ -234,10 +241,8 @@ class DataSmart:
234 Create a copy of self by setting _data to self 241 Create a copy of self by setting _data to self
235 """ 242 """
236 # we really want this to be a DataSmart... 243 # we really want this to be a DataSmart...
237 data = DataSmart() 244 data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy())
238 data.dict["_data"] = self.dict 245 data.dict["_data"] = self.dict
239 data._seen_overrides = copy.deepcopy(self._seen_overrides)
240 data._special_values = copy.deepcopy(self._special_values)
241 246
242 return data 247 return data
243 248
diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py
index 7ab0590..24aebc4 100644
--- a/bitbake/lib/bb/fetch/__init__.py
+++ b/bitbake/lib/bb/fetch/__init__.py
@@ -38,13 +38,16 @@ class NoMethodError(Exception):
38class MissingParameterError(Exception): 38class MissingParameterError(Exception):
39 """Exception raised when a fetch method is missing a critical parameter in the url""" 39 """Exception raised when a fetch method is missing a critical parameter in the url"""
40 40
41class ParameterError(Exception):
42 """Exception raised when a url cannot be proccessed due to invalid parameters."""
43
41class MD5SumError(Exception): 44class MD5SumError(Exception):
42 """Exception raised when a MD5SUM of a file does not match the expected one""" 45 """Exception raised when a MD5SUM of a file does not match the expected one"""
43 46
44def uri_replace(uri, uri_find, uri_replace, d): 47def uri_replace(uri, uri_find, uri_replace, d):
45# bb.note("uri_replace: operating on %s" % uri) 48# bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: operating on %s" % uri)
46 if not uri or not uri_find or not uri_replace: 49 if not uri or not uri_find or not uri_replace:
47 bb.debug(1, "uri_replace: passed an undefined value, not replacing") 50 bb.msg.debug(1, bb.msg.domain.Fetcher, "uri_replace: passed an undefined value, not replacing")
48 uri_decoded = list(bb.decodeurl(uri)) 51 uri_decoded = list(bb.decodeurl(uri))
49 uri_find_decoded = list(bb.decodeurl(uri_find)) 52 uri_find_decoded = list(bb.decodeurl(uri_find))
50 uri_replace_decoded = list(bb.decodeurl(uri_replace)) 53 uri_replace_decoded = list(bb.decodeurl(uri_replace))
@@ -62,9 +65,9 @@ def uri_replace(uri, uri_find, uri_replace, d):
62 localfn = bb.fetch.localpath(uri, d) 65 localfn = bb.fetch.localpath(uri, d)
63 if localfn: 66 if localfn:
64 result_decoded[loc] = os.path.dirname(result_decoded[loc]) + "/" + os.path.basename(bb.fetch.localpath(uri, d)) 67 result_decoded[loc] = os.path.dirname(result_decoded[loc]) + "/" + os.path.basename(bb.fetch.localpath(uri, d))
65# bb.note("uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc])) 68# bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc]))
66 else: 69 else:
67# bb.note("uri_replace: no match") 70# bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: no match")
68 return uri 71 return uri
69# else: 72# else:
70# for j in i.keys(): 73# for j in i.keys():
@@ -72,62 +75,94 @@ def uri_replace(uri, uri_find, uri_replace, d):
72 return bb.encodeurl(result_decoded) 75 return bb.encodeurl(result_decoded)
73 76
74methods = [] 77methods = []
78urldata = {}
75 79
76def init(urls = [], d = None): 80def init(urls = [], d = None):
77 if d == None: 81 if d == None:
78 bb.debug(2,"BUG init called with None as data object!!!") 82 bb.msg.debug(2, bb.msg.domain.Fetcher, "BUG init called with None as data object!!!")
79 return 83 return
80 84
81 for m in methods: 85 for m in methods:
82 m.urls = [] 86 m.urls = []
83 87
84 for u in urls: 88 for u in urls:
89 ud = initdata(u, d)
90 if ud.method:
91 ud.method.urls.append(u)
92
93def initdata(url, d):
94 if url not in urldata:
95 ud = FetchData()
96 (ud.type, ud.host, ud.path, ud.user, ud.pswd, ud.parm) = bb.decodeurl(data.expand(url, d))
97 ud.date = Fetch.getSRCDate(d)
85 for m in methods: 98 for m in methods:
86 m.data = d 99 if m.supports(url, ud, d):
87 if m.supports(u, d): 100 ud.localpath = m.localpath(url, ud, d)
88 m.urls.append(u) 101 ud.md5 = ud.localpath + '.md5'
102 # if user sets localpath for file, use it instead.
103 if "localpath" in ud.parm:
104 ud.localpath = ud.parm["localpath"]
105 ud.method = m
106 break
107 urldata[url] = ud
108 return urldata[url]
89 109
90def go(d): 110def go(d):
91 """Fetch all urls""" 111 """Fetch all urls"""
92 for m in methods: 112 for m in methods:
93 if m.urls: 113 for u in m.urls:
94 m.go(d) 114 ud = urldata[u]
115 if ud.localfile and not m.forcefetch(u, ud, d) and os.path.exists(urldata[u].md5):
116 # File already present along with md5 stamp file
117 # Touch md5 file to show activity
118 os.utime(ud.md5, None)
119 continue
120 # RP - is olddir needed?
121 # olddir = os.path.abspath(os.getcwd())
122 m.go(u, ud , d)
123 # os.chdir(olddir)
124 if ud.localfile and not m.forcefetch(u, ud, d):
125 Fetch.write_md5sum(u, ud, d)
95 126
96def localpaths(d): 127def localpaths(d):
97 """Return a list of the local filenames, assuming successful fetch""" 128 """Return a list of the local filenames, assuming successful fetch"""
98 local = [] 129 local = []
99 for m in methods: 130 for m in methods:
100 for u in m.urls: 131 for u in m.urls:
101 local.append(m.localpath(u, d)) 132 local.append(urldata[u].localpath)
102 return local 133 return local
103 134
104def localpath(url, d): 135def localpath(url, d):
105 for m in methods: 136 ud = initdata(url, d)
106 if m.supports(url, d): 137 if ud.method:
107 return m.localpath(url, d) 138 return ud.localpath
108 return url 139 return url
109 140
141class FetchData(object):
142 """Class for fetcher variable store"""
143 def __init__(self):
144 self.localfile = ""
145
146
110class Fetch(object): 147class Fetch(object):
111 """Base class for 'fetch'ing data""" 148 """Base class for 'fetch'ing data"""
112 149
113 def __init__(self, urls = []): 150 def __init__(self, urls = []):
114 self.urls = [] 151 self.urls = []
115 for url in urls:
116 if self.supports(bb.decodeurl(url), d) is 1:
117 self.urls.append(url)
118 152
119 def supports(url, d): 153 def supports(self, url, urldata, d):
120 """Check to see if this fetch class supports a given url. 154 """
121 Expects supplied url in list form, as outputted by bb.decodeurl(). 155 Check to see if this fetch class supports a given url.
122 """ 156 """
123 return 0 157 return 0
124 supports = staticmethod(supports)
125 158
126 def localpath(url, d): 159 def localpath(self, url, urldata, d):
127 """Return the local filename of a given url assuming a successful fetch. 160 """
161 Return the local filename of a given url assuming a successful fetch.
162 Can also setup variables in urldata for use in go (saving code duplication
163 and duplicate code execution)
128 """ 164 """
129 return url 165 return url
130 localpath = staticmethod(localpath)
131 166
132 def setUrls(self, urls): 167 def setUrls(self, urls):
133 self.__urls = urls 168 self.__urls = urls
@@ -137,16 +172,17 @@ class Fetch(object):
137 172
138 urls = property(getUrls, setUrls, None, "Urls property") 173 urls = property(getUrls, setUrls, None, "Urls property")
139 174
140 def setData(self, data): 175 def forcefetch(self, url, urldata, d):
141 self.__data = data 176 """
142 177 Force a fetch, even if localpath exists?
143 def getData(self): 178 """
144 return self.__data 179 return False
145
146 data = property(getData, setData, None, "Data property")
147 180
148 def go(self, urls = []): 181 def go(self, url, urldata, d):
149 """Fetch urls""" 182 """
183 Fetch urls
184 Assumes localpath was called first
185 """
150 raise NoMethodError("Missing implementation for url") 186 raise NoMethodError("Missing implementation for url")
151 187
152 def getSRCDate(d): 188 def getSRCDate(d):
@@ -155,7 +191,12 @@ class Fetch(object):
155 191
156 d the bb.data module 192 d the bb.data module
157 """ 193 """
158 return data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1 ) 194 pn = data.getVar("PN", d, 1)
195
196 if pn:
197 return data.getVar("SRCDATE_%s" % pn, d, 1) or data.getVar("CVSDATE_%s" % pn, d, 1) or data.getVar("DATE", d, 1)
198
199 return data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
159 getSRCDate = staticmethod(getSRCDate) 200 getSRCDate = staticmethod(getSRCDate)
160 201
161 def try_mirror(d, tarfn): 202 def try_mirror(d, tarfn):
@@ -168,6 +209,11 @@ class Fetch(object):
168 d Is a bb.data instance 209 d Is a bb.data instance
169 tarfn is the name of the tarball 210 tarfn is the name of the tarball
170 """ 211 """
212 tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn)
213 if os.access(tarpath, os.R_OK):
214 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % tarfn)
215 return True
216
171 pn = data.getVar('PN', d, True) 217 pn = data.getVar('PN', d, True)
172 src_tarball_stash = None 218 src_tarball_stash = None
173 if pn: 219 if pn:
@@ -176,36 +222,45 @@ class Fetch(object):
176 for stash in src_tarball_stash: 222 for stash in src_tarball_stash:
177 fetchcmd = data.getVar("FETCHCOMMAND_mirror", d, True) or data.getVar("FETCHCOMMAND_wget", d, True) 223 fetchcmd = data.getVar("FETCHCOMMAND_mirror", d, True) or data.getVar("FETCHCOMMAND_wget", d, True)
178 uri = stash + tarfn 224 uri = stash + tarfn
179 bb.note("fetch " + uri) 225 bb.msg.note(1, bb.msg.domain.Fetcher, "fetch " + uri)
180 fetchcmd = fetchcmd.replace("${URI}", uri) 226 fetchcmd = fetchcmd.replace("${URI}", uri)
181 ret = os.system(fetchcmd) 227 ret = os.system(fetchcmd)
182 if ret == 0: 228 if ret == 0:
183 bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn) 229 bb.msg.note(1, bb.msg.domain.Fetcher, "Fetched %s from tarball stash, skipping checkout" % tarfn)
184 return True 230 return True
185 return False 231 return False
186 try_mirror = staticmethod(try_mirror) 232 try_mirror = staticmethod(try_mirror)
187 233
188 def check_for_tarball(d, tarfn, dldir, date): 234 def verify_md5sum(ud, got_sum):
189 """ 235 """
190 Check for a local copy then check the tarball stash. 236 Verify the md5sum we wanted with the one we got
191 Both checks are skipped if date == 'now'.
192
193 d Is a bb.data instance
194 tarfn is the name of the tarball
195 date is the SRCDATE
196 """ 237 """
197 if "now" != date: 238 wanted_sum = None
198 dl = os.path.join(dldir, tarfn) 239 if 'md5sum' in ud.parm:
199 if os.access(dl, os.R_OK): 240 wanted_sum = ud.parm['md5sum']
200 bb.debug(1, "%s already exists, skipping checkout." % tarfn) 241 if not wanted_sum:
201 return True 242 return True
202 243
203 # try to use the tarball stash 244 return wanted_sum == got_sum
204 if Fetch.try_mirror(d, tarfn): 245 verify_md5sum = staticmethod(verify_md5sum)
205 return True 246
206 return False 247 def write_md5sum(url, ud, d):
207 check_for_tarball = staticmethod(check_for_tarball) 248 if bb.which(data.getVar('PATH', d), 'md5sum'):
208 249 try:
250 md5pipe = os.popen('md5sum ' + ud.localpath)
251 md5data = (md5pipe.readline().split() or [ "" ])[0]
252 md5pipe.close()
253 except OSError:
254 md5data = ""
255
256 # verify the md5sum
257 if not Fetch.verify_md5sum(ud, md5data):
258 raise MD5SumError(url)
259
260 md5out = file(ud.md5, 'w')
261 md5out.write(md5data)
262 md5out.close()
263 write_md5sum = staticmethod(write_md5sum)
209 264
210import cvs 265import cvs
211import git 266import git
@@ -214,6 +269,7 @@ import svn
214import wget 269import wget
215import svk 270import svk
216import ssh 271import ssh
272import perforce
217 273
218methods.append(cvs.Cvs()) 274methods.append(cvs.Cvs())
219methods.append(git.Git()) 275methods.append(git.Git())
@@ -222,3 +278,4 @@ methods.append(svn.Svn())
222methods.append(wget.Wget()) 278methods.append(wget.Wget())
223methods.append(svk.Svk()) 279methods.append(svk.Svk())
224methods.append(ssh.SSH()) 280methods.append(ssh.SSH())
281methods.append(perforce.Perforce())
diff --git a/bitbake/lib/bb/fetch/cvs.py b/bitbake/lib/bb/fetch/cvs.py
index 0b24775..3bdac17 100644
--- a/bitbake/lib/bb/fetch/cvs.py
+++ b/bitbake/lib/bb/fetch/cvs.py
@@ -33,164 +33,119 @@ from bb.fetch import FetchError
33from bb.fetch import MissingParameterError 33from bb.fetch import MissingParameterError
34 34
35class Cvs(Fetch): 35class Cvs(Fetch):
36 """Class to fetch a module or modules from cvs repositories""" 36 """
37 def supports(url, d): 37 Class to fetch a module or modules from cvs repositories
38 """Check to see if a given url can be fetched with cvs. 38 """
39 Expects supplied url in list form, as outputted by bb.decodeurl(). 39 def supports(self, url, ud, d):
40 """ 40 """
41 (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) 41 Check to see if a given url can be fetched with cvs.
42 return type in ['cvs', 'pserver'] 42 """
43 supports = staticmethod(supports) 43 return ud.type in ['cvs', 'pserver']
44
45 def localpath(url, d):
46 (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
47 if "localpath" in parm:
48# if user overrides local path, use it.
49 return parm["localpath"]
50 44
51 if not "module" in parm: 45 def localpath(self, url, ud, d):
46 if not "module" in ud.parm:
52 raise MissingParameterError("cvs method needs a 'module' parameter") 47 raise MissingParameterError("cvs method needs a 'module' parameter")
53 else: 48 ud.module = ud.parm["module"]
54 module = parm["module"] 49
55 if 'tag' in parm: 50 ud.tag = ""
56 tag = parm['tag'] 51 if 'tag' in ud.parm:
57 else: 52 ud.tag = ud.parm['tag']
58 tag = "" 53
59 if 'date' in parm: 54 # Override the default date in certain cases
60 date = parm['date'] 55 if 'date' in ud.parm:
61 else: 56 ud.date = ud.parm['date']
62 if not tag: 57 elif ud.tag:
63 date = Fetch.getSRCDate(d) 58 ud.date = ""
64 else: 59
65 date = "" 60 ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date), d)
61
62 return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
66 63
67 return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, tag, date), d)) 64 def forcefetch(self, url, ud, d):
68 localpath = staticmethod(localpath) 65 if (ud.date == "now"):
66 return True
67 return False
69 68
70 def go(self, d, urls = []): 69 def go(self, loc, ud, d):
71 """Fetch urls""" 70
72 if not urls: 71 # try to use the tarball stash
73 urls = self.urls 72 if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
73 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping cvs checkout." % ud.localpath)
74 return
75
76 method = "pserver"
77 if "method" in ud.parm:
78 method = ud.parm["method"]
79
80 localdir = ud.module
81 if "localdir" in ud.parm:
82 localdir = ud.parm["localdir"]
83
84 cvs_rsh = None
85 if method == "ext":
86 if "rsh" in ud.parm:
87 cvs_rsh = ud.parm["rsh"]
88
89 if method == "dir":
90 cvsroot = ud.path
91 else:
92 cvsroot = ":" + method + ":" + ud.user
93 if ud.pswd:
94 cvsroot += ":" + ud.pswd
95 cvsroot += "@" + ud.host + ":" + ud.path
96
97 options = []
98 if ud.date:
99 options.append("-D %s" % ud.date)
100 if ud.tag:
101 options.append("-r %s" % ud.tag)
74 102
75 localdata = data.createCopy(d) 103 localdata = data.createCopy(d)
76 data.setVar('OVERRIDES', "cvs:%s" % data.getVar('OVERRIDES', localdata), localdata) 104 data.setVar('OVERRIDES', "cvs:%s" % data.getVar('OVERRIDES', localdata), localdata)
77 data.update_data(localdata) 105 data.update_data(localdata)
78 106
79 for loc in urls: 107 data.setVar('CVSROOT', cvsroot, localdata)
80 (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, localdata)) 108 data.setVar('CVSCOOPTS', " ".join(options), localdata)
81 if not "module" in parm: 109 data.setVar('CVSMODULE', ud.module, localdata)
82 raise MissingParameterError("cvs method needs a 'module' parameter") 110 cvscmd = data.getVar('FETCHCOMMAND', localdata, 1)
83 else: 111 cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, 1)
84 module = parm["module"] 112
85 113 if cvs_rsh:
86 dlfile = self.localpath(loc, localdata) 114 cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd)
87 dldir = data.getVar('DL_DIR', localdata, 1) 115 cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd)
88# if local path contains the cvs 116
89# module, consider the dir above it to be the 117 # create module directory
90# download directory 118 bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory")
91# pos = dlfile.find(module) 119 pkg = data.expand('${PN}', d)
92# if pos: 120 pkgdir = os.path.join(data.expand('${CVSDIR}', localdata), pkg)
93# dldir = dlfile[:pos] 121 moddir = os.path.join(pkgdir,localdir)
94# else: 122 if os.access(os.path.join(moddir,'CVS'), os.R_OK):
95# dldir = os.path.dirname(dlfile) 123 bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc)
96 124 # update sources there
97# setup cvs options
98 options = []
99 if 'tag' in parm:
100 tag = parm['tag']
101 else:
102 tag = ""
103
104 if 'date' in parm:
105 date = parm['date']
106 else:
107 if not tag:
108 date = Fetch.getSRCDate(d)
109 else:
110 date = ""
111
112 if "method" in parm:
113 method = parm["method"]
114 else:
115 method = "pserver"
116
117 if "localdir" in parm:
118 localdir = parm["localdir"]
119 else:
120 localdir = module
121
122 cvs_rsh = None
123 if method == "ext":
124 if "rsh" in parm:
125 cvs_rsh = parm["rsh"]
126
127 tarfn = data.expand('%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, tag, date), localdata)
128 data.setVar('TARFILES', dlfile, localdata)
129 data.setVar('TARFN', tarfn, localdata)
130
131 if Fetch.check_for_tarball(d, tarfn, dldir, date):
132 continue
133
134 if date:
135 options.append("-D %s" % date)
136 if tag:
137 options.append("-r %s" % tag)
138
139 olddir = os.path.abspath(os.getcwd())
140 os.chdir(data.expand(dldir, localdata))
141
142# setup cvsroot
143 if method == "dir":
144 cvsroot = path
145 else:
146 cvsroot = ":" + method + ":" + user
147 if pswd:
148 cvsroot += ":" + pswd
149 cvsroot += "@" + host + ":" + path
150
151 data.setVar('CVSROOT', cvsroot, localdata)
152 data.setVar('CVSCOOPTS', " ".join(options), localdata)
153 data.setVar('CVSMODULE', module, localdata)
154 cvscmd = data.getVar('FETCHCOMMAND', localdata, 1)
155 cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, 1)
156
157 if cvs_rsh:
158 cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd)
159 cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd)
160
161# create module directory
162 bb.debug(2, "Fetch: checking for module directory")
163 pkg=data.expand('${PN}', d)
164 pkgdir=os.path.join(data.expand('${CVSDIR}', localdata), pkg)
165 moddir=os.path.join(pkgdir,localdir)
166 if os.access(os.path.join(moddir,'CVS'), os.R_OK):
167 bb.note("Update " + loc)
168# update sources there
169 os.chdir(moddir)
170 myret = os.system(cvsupdatecmd)
171 else:
172 bb.note("Fetch " + loc)
173# check out sources there
174 bb.mkdirhier(pkgdir)
175 os.chdir(pkgdir)
176 bb.debug(1, "Running %s" % cvscmd)
177 myret = os.system(cvscmd)
178
179 if myret != 0 or not os.access(moddir, os.R_OK):
180 try:
181 os.rmdir(moddir)
182 except OSError:
183 pass
184 raise FetchError(module)
185
186 os.chdir(moddir) 125 os.chdir(moddir)
187 os.chdir('..') 126 myret = os.system(cvsupdatecmd)
188# tar them up to a defined filename 127 else:
189 myret = os.system("tar -czf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(moddir))) 128 bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
190 if myret != 0: 129 # check out sources there
191 try: 130 bb.mkdirhier(pkgdir)
192 os.unlink(tarfn) 131 os.chdir(pkgdir)
193 except OSError: 132 bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % cvscmd)
194 pass 133 myret = os.system(cvscmd)
195 os.chdir(olddir) 134
196 del localdata 135 if myret != 0 or not os.access(moddir, os.R_OK):
136 try:
137 os.rmdir(moddir)
138 except OSError:
139 pass
140 raise FetchError(ud.module)
141
142 os.chdir(moddir)
143 os.chdir('..')
144 # tar them up to a defined filename
145 myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(moddir)))
146 if myret != 0:
147 try:
148 os.unlink(ud.localpath)
149 except OSError:
150 pass
151 raise FetchError(ud.module)
diff --git a/bitbake/lib/bb/fetch/git.py b/bitbake/lib/bb/fetch/git.py
index 49235c1..75a7629 100644
--- a/bitbake/lib/bb/fetch/git.py
+++ b/bitbake/lib/bb/fetch/git.py
@@ -37,7 +37,7 @@ def prunedir(topdir):
37 37
38def rungitcmd(cmd,d): 38def rungitcmd(cmd,d):
39 39
40 bb.debug(1, "Running %s" % cmd) 40 bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % cmd)
41 41
42 # Need to export PATH as git is likely to be in metadata paths 42 # Need to export PATH as git is likely to be in metadata paths
43 # rather than host provided 43 # rather than host provided
@@ -48,108 +48,80 @@ def rungitcmd(cmd,d):
48 if myret != 0: 48 if myret != 0:
49 raise FetchError("Git: %s failed" % pathcmd) 49 raise FetchError("Git: %s failed" % pathcmd)
50 50
51def gettag(parm):
52 if 'tag' in parm:
53 tag = parm['tag']
54 else:
55 tag = ""
56 if not tag:
57 tag = "master"
58
59 return tag
60
61def getprotocol(parm):
62 if 'protocol' in parm:
63 proto = parm['protocol']
64 else:
65 proto = ""
66 if not proto:
67 proto = "rsync"
68
69 return proto
70
71def localfile(url, d):
72 """Return the filename to cache the checkout in"""
73 (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
74
75 #if user sets localpath for file, use it instead.
76 if "localpath" in parm:
77 return parm["localpath"]
78
79 tag = gettag(parm)
80
81 return data.expand('git_%s%s_%s.tar.gz' % (host, path.replace('/', '.'), tag), d)
82
83class Git(Fetch): 51class Git(Fetch):
84 """Class to fetch a module or modules from git repositories""" 52 """Class to fetch a module or modules from git repositories"""
85 def supports(url, d): 53 def supports(self, url, ud, d):
86 """Check to see if a given url can be fetched with cvs. 54 """
87 Expects supplied url in list form, as outputted by bb.decodeurl(). 55 Check to see if a given url can be fetched with cvs.
88 """ 56 """
89 (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) 57 return ud.type in ['git']
90 return type in ['git']
91 supports = staticmethod(supports)
92 58
93 def localpath(url, d): 59 def localpath(self, url, ud, d):
94 60
95 return os.path.join(data.getVar("DL_DIR", d, 1), localfile(url, d)) 61 ud.proto = "rsync"
62 if 'protocol' in ud.parm:
63 ud.proto = ud.parm['protocol']
96 64
97 localpath = staticmethod(localpath) 65 ud.tag = "master"
66 if 'tag' in ud.parm:
67 ud.tag = ud.parm['tag']
98 68
99 def go(self, d, urls = []): 69 ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.tag), d)
100 """Fetch urls"""
101 if not urls:
102 urls = self.urls
103 70
104 for loc in urls: 71 return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
105 (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, d))
106 72
107 tag = gettag(parm) 73 def forcefetch(self, url, ud, d):
108 proto = getprotocol(parm) 74 # tag=="master" must always update
75 if (ud.tag == "master"):
76 return True
77 return False
109 78
110 gitsrcname = '%s%s' % (host, path.replace('/', '.')) 79 def go(self, loc, ud, d):
80 """Fetch url"""
111 81
112 repofilename = 'git_%s.tar.gz' % (gitsrcname) 82 if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
113 repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename) 83 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath)
114 repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname) 84 return
115 85
116 coname = '%s' % (tag) 86 gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.'))
117 codir = os.path.join(repodir, coname)
118 87
119 cofile = self.localpath(loc, d) 88 repofilename = 'git_%s.tar.gz' % (gitsrcname)
89 repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename)
90 repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
120 91
121 # tag=="master" must always update 92 coname = '%s' % (ud.tag)
122 if (tag != "master") and Fetch.try_mirror(d, localfile(loc, d)): 93 codir = os.path.join(repodir, coname)
123 bb.debug(1, "%s already exists (or was stashed). Skipping git checkout." % cofile)
124 continue
125 94
126 if not os.path.exists(repodir): 95 if not os.path.exists(repodir):
127 if Fetch.try_mirror(d, repofilename): 96 if Fetch.try_mirror(d, repofilename):
128 bb.mkdirhier(repodir) 97 bb.mkdirhier(repodir)
129 os.chdir(repodir) 98 os.chdir(repodir)
130 rungitcmd("tar -xzf %s" % (repofile),d) 99 rungitcmd("tar -xzf %s" % (repofile),d)
131 else: 100 else:
132 rungitcmd("git clone -n %s://%s%s %s" % (proto, host, path, repodir),d) 101 rungitcmd("git clone -n %s://%s%s %s" % (ud.proto, ud.host, ud.path, repodir),d)
133 102
134 os.chdir(repodir) 103 os.chdir(repodir)
135 rungitcmd("git pull %s://%s%s" % (proto, host, path),d) 104 rungitcmd("git pull %s://%s%s" % (ud.proto, ud.host, ud.path),d)
136 rungitcmd("git pull --tags %s://%s%s" % (proto, host, path),d) 105 rungitcmd("git pull --tags %s://%s%s" % (ud.proto, ud.host, ud.path),d)
137 rungitcmd("git prune-packed", d) 106 rungitcmd("git prune-packed", d)
138 # old method of downloading tags 107 rungitcmd("git pack-redundant --all | xargs -r rm", d)
139 #rungitcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (host, path, os.path.join(repodir, ".git", "")),d) 108 # Remove all but the .git directory
109 rungitcmd("rm * -Rf", d)
110 # old method of downloading tags
111 #rungitcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (ud.host, ud.path, os.path.join(repodir, ".git", "")),d)
140 112
141 os.chdir(repodir) 113 os.chdir(repodir)
142 bb.note("Creating tarball of git repository") 114 bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository")