summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/cooker.py
diff options
context:
space:
mode:
authorRichard Purdie <richard@openedhand.com>2007-01-08 23:53:01 +0000
committerRichard Purdie <richard@openedhand.com>2007-01-08 23:53:01 +0000
commitf5665d5bfcfb13d01da9e4c7d5046453e80f7baf (patch)
treeb8908549afaf3006bf3763419711090ac999c2a4 /bitbake/lib/bb/cooker.py
parentaec95de5f7dca2afa3a4a0bdb0d4d553c13f680d (diff)
downloadpoky-f5665d5bfcfb13d01da9e4c7d5046453e80f7baf.tar.gz
bitbake: Sync with upstream.
* File licence headers were sanitised causing most of the diff. * cooker.py was created from bin/bitbake. * cvs fetcher port option was added * The -f force option was fixed to work correctly * Multiple entries in rrecrdeps are now handled correctly (allows adding do_deploy to image depends) git-svn-id: https://svn.o-hand.com/repos/poky/trunk@1129 311d38ba-8fff-0310-9ca6-ca027cbcb966
Diffstat (limited to 'bitbake/lib/bb/cooker.py')
-rw-r--r--bitbake/lib/bb/cooker.py665
1 files changed, 665 insertions, 0 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
new file mode 100644
index 0000000000..8a9c588633
--- /dev/null
+++ b/bitbake/lib/bb/cooker.py
@@ -0,0 +1,665 @@
1#!/usr/bin/env python
2# ex:ts=4:sw=4:sts=4:et
3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4#
5# Copyright (C) 2003, 2004 Chris Larson
6# Copyright (C) 2003, 2004 Phil Blundell
7# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8# Copyright (C) 2005 Holger Hans Peter Freyther
9# Copyright (C) 2005 ROAD GmbH
10# Copyright (C) 2006 Richard Purdie
11#
12# This program is free software; you can redistribute it and/or modify
13# it under the terms of the GNU General Public License version 2 as
14# published by the Free Software Foundation.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License along
22# with this program; if not, write to the Free Software Foundation, Inc.,
23# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25import sys, os, getopt, glob, copy, os.path, re, time
26import bb
27from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
28from sets import Set
29import itertools
30
31parsespin = itertools.cycle( r'|/-\\' )
32
33#============================================================================#
34# BBStatistics
35#============================================================================#
36class BBStatistics:
37 """
38 Manage build statistics for one run
39 """
40 def __init__(self ):
41 self.attempt = 0
42 self.success = 0
43 self.fail = 0
44 self.deps = 0
45
46 def show( self ):
47 print "Build statistics:"
48 print " Attempted builds: %d" % self.attempt
49 if self.fail:
50 print " Failed builds: %d" % self.fail
51 if self.deps:
52 print " Dependencies not satisfied: %d" % self.deps
53 if self.fail or self.deps: return 1
54 else: return 0
55
56#============================================================================#
57# BBCooker
58#============================================================================#
59class BBCooker:
60 """
61 Manages one bitbake build run
62 """
63
64 Statistics = BBStatistics # make it visible from the shell
65
66 def __init__( self ):
67 self.build_cache_fail = []
68 self.build_cache = []
69 self.stats = BBStatistics()
70 self.status = None
71
72 self.cache = None
73 self.bb_cache = None
74
75 def tryBuildPackage(self, fn, item, task, the_data, build_depends):
76 """
77 Build one task of a package, optionally build following task depends
78 """
79 bb.event.fire(bb.event.PkgStarted(item, the_data))
80 try:
81 self.stats.attempt += 1
82 if not build_depends:
83 bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data)
84 if not self.configuration.dry_run:
85 bb.build.exec_task('do_%s' % task, the_data)
86 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
87 self.build_cache.append(fn)
88 return True
89 except bb.build.FuncFailed:
90 self.stats.fail += 1
91 bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
92 bb.event.fire(bb.event.PkgFailed(item, the_data))
93 self.build_cache_fail.append(fn)
94 raise
95 except bb.build.EventException, e:
96 self.stats.fail += 1
97 event = e.args[1]
98 bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
99 bb.event.fire(bb.event.PkgFailed(item, the_data))
100 self.build_cache_fail.append(fn)
101 raise
102
103 def tryBuild( self, fn, build_depends):
104 """
105 Build a provider and its dependencies.
106 build_depends is a list of previous build dependencies (not runtime)
107 If build_depends is empty, we're dealing with a runtime depends
108 """
109
110 the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
111
112 item = self.status.pkg_fn[fn]
113
114 if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
115 self.build_cache.append(fn)
116 return True
117
118 return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends)
119
120 def showVersions( self ):
121 pkg_pn = self.status.pkg_pn
122 preferred_versions = {}
123 latest_versions = {}
124
125 # Sort by priority
126 for pn in pkg_pn.keys():
127 (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
128 preferred_versions[pn] = (pref_ver, pref_file)
129 latest_versions[pn] = (last_ver, last_file)
130
131 pkg_list = pkg_pn.keys()
132 pkg_list.sort()
133
134 for p in pkg_list:
135 pref = preferred_versions[p]
136 latest = latest_versions[p]
137
138 if pref != latest:
139 prefstr = pref[0][0] + "-" + pref[0][1]
140 else:
141 prefstr = ""
142
143 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
144 prefstr)
145
146
147 def showEnvironment( self ):
148 """Show the outer or per-package environment"""
149 if self.configuration.buildfile:
150 self.cb = None
151 self.bb_cache = bb.cache.init(self)
152 try:
153 self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self.configuration.data)
154 except IOError, e:
155 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % ( self.configuration.buildfile, e ))
156 except Exception, e:
157 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
158 # emit variables and shell functions
159 try:
160 data.update_data( self.configuration.data )
161 data.emit_env(sys.__stdout__, self.configuration.data, True)
162 except Exception, e:
163 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
164 # emit the metadata which isnt valid shell
165 data.expandKeys( self.configuration.data )
166 for e in self.configuration.data.keys():
167 if data.getVarFlag( e, 'python', self.configuration.data ):
168 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
169
170 def generateDotGraph( self, pkgs_to_build, ignore_deps ):
171 """
172 Generate a task dependency graph.
173
174 pkgs_to_build A list of packages that needs to be built
175 ignore_deps A list of names where processing of dependencies
176 should be stopped. e.g. dependencies that get
177 """
178
179 for dep in ignore_deps:
180 self.status.ignored_dependencies.add(dep)
181
182 localdata = data.createCopy(self.configuration.data)
183 bb.data.update_data(localdata)
184 bb.data.expandKeys(localdata)
185 taskdata = bb.taskdata.TaskData(self.configuration.abort)
186
187 runlist = []
188 try:
189 for k in pkgs_to_build:
190 taskdata.add_provider(localdata, self.status, k)
191 runlist.append([k, "do_%s" % self.configuration.cmd])
192 taskdata.add_unresolved(localdata, self.status)
193 except bb.providers.NoProvider:
194 sys.exit(1)
195 rq = bb.runqueue.RunQueue()
196 rq.prepare_runqueue(self, self.configuration.data, self.status, taskdata, runlist)
197
198 seen_fnids = []
199 depends_file = file('depends.dot', 'w' )
200 tdepends_file = file('task-depends.dot', 'w' )
201 print >> depends_file, "digraph depends {"
202 print >> tdepends_file, "digraph depends {"
203 rq.prio_map.reverse()
204 for task1 in range(len(rq.runq_fnid)):
205 task = rq.prio_map[task1]
206 taskname = rq.runq_task[task]
207 fnid = rq.runq_fnid[task]
208 fn = taskdata.fn_index[fnid]
209 pn = self.status.pkg_fn[fn]
210 version = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
211 print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
212 for dep in rq.runq_depends[task]:
213 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
214 deppn = self.status.pkg_fn[depfn]
215 print >> tdepends_file, '"%s.%s" -> "%s.%s"' % (pn, rq.runq_task[task], deppn, rq.runq_task[dep])
216 if fnid not in seen_fnids:
217 seen_fnids.append(fnid)
218 packages = []
219 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
220 for depend in self.status.deps[fn]:
221 print >> depends_file, '"%s" -> "%s"' % (pn, depend)
222 rdepends = self.status.rundeps[fn]
223 for package in rdepends:
224 for rdepend in rdepends[package]:
225 print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
226 packages.append(package)
227 rrecs = self.status.runrecs[fn]
228 for package in rrecs:
229 for rdepend in rrecs[package]:
230 print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
231 if not package in packages:
232 packages.append(package)
233 for package in packages:
234 if package != pn:
235 print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
236 for depend in self.status.deps[fn]:
237 print >> depends_file, '"%s" -> "%s"' % (package, depend)
238 # Prints a flattened form of the above where subpackages of a package are merged into the main pn
239 #print >> depends_file, '"%s" [label="%s %s\\n%s\\n%s"]' % (pn, pn, taskname, version, fn)
240 #for rdep in taskdata.rdepids[fnid]:
241 # print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, taskdata.run_names_index[rdep])
242 #for dep in taskdata.depids[fnid]:
243 # print >> depends_file, '"%s" -> "%s"' % (pn, taskdata.build_names_index[dep])
244 print >> depends_file, "}"
245 print >> tdepends_file, "}"
246 bb.msg.note(1, bb.msg.domain.Collection, "Dependencies saved to 'depends.dot'")
247 bb.msg.note(1, bb.msg.domain.Collection, "Task dependencies saved to 'task-depends.dot'")
248
249 def buildDepgraph( self ):
250 all_depends = self.status.all_depends
251 pn_provides = self.status.pn_provides
252
253 localdata = data.createCopy(self.configuration.data)
254 bb.data.update_data(localdata)
255 bb.data.expandKeys(localdata)
256
257 def calc_bbfile_priority(filename):
258 for (regex, pri) in self.status.bbfile_config_priorities:
259 if regex.match(filename):
260 return pri
261 return 0
262
263 # Handle PREFERRED_PROVIDERS
264 for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
265 (providee, provider) = p.split(':')
266 if providee in self.status.preferred and self.status.preferred[providee] != provider:
267 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
268 self.status.preferred[providee] = provider
269
270 # Calculate priorities for each file
271 for p in self.status.pkg_fn.keys():
272 self.status.bbfile_priority[p] = calc_bbfile_priority(p)
273
274 def buildWorldTargetList(self):
275 """
276 Build package list for "bitbake world"
277 """
278 all_depends = self.status.all_depends
279 pn_provides = self.status.pn_provides
280 bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
281 for f in self.status.possible_world:
282 terminal = True
283 pn = self.status.pkg_fn[f]
284
285 for p in pn_provides[pn]:
286 if p.startswith('virtual/'):
287 bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
288 terminal = False
289 break
290 for pf in self.status.providers[p]:
291 if self.status.pkg_fn[pf] != pn:
292 bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
293 terminal = False
294 break
295 if terminal:
296 self.status.world_target.add(pn)
297
298 # drop reference count now
299 self.status.possible_world = None
300 self.status.all_depends = None
301
302 def myProgressCallback( self, x, y, f, from_cache ):
303 """Update any tty with the progress change"""
304 if os.isatty(sys.stdout.fileno()):
305 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
306 sys.stdout.flush()
307 else:
308 if x == 1:
309 sys.stdout.write("Parsing .bb files, please wait...")
310 sys.stdout.flush()
311 if x == y:
312 sys.stdout.write("done.")
313 sys.stdout.flush()
314
315 def interactiveMode( self ):
316 """Drop off into a shell"""
317 try:
318 from bb import shell
319 except ImportError, details:
320 bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
321 else:
322 bb.data.update_data( self.configuration.data )
323 bb.data.expandKeys( self.configuration.data )
324 shell.start( self )
325 sys.exit( 0 )
326
327 def parseConfigurationFile( self, afile ):
328 try:
329 self.configuration.data = bb.parse.handle( afile, self.configuration.data )
330
331 # Add the handlers we inherited by INHERIT
332 # we need to do this manually as it is not guranteed
333 # we will pick up these classes... as we only INHERIT
334 # on .inc and .bb files but not on .conf
335 data = bb.data.createCopy( self.configuration.data )
336 inherits = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
337 for inherit in inherits:
338 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
339
340 # FIXME: This assumes that we included at least one .inc file
341 for var in bb.data.keys(data):
342 if bb.data.getVarFlag(var, 'handler', data):
343 bb.event.register(var,bb.data.getVar(var, data))
344
345 except IOError:
346 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % afile )
347 except bb.parse.ParseError, details:
348 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
349
350 def handleCollections( self, collections ):
351 """Handle collections"""
352 if collections:
353 collection_list = collections.split()
354 for c in collection_list:
355 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
356 if regex == None:
357 bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
358 continue
359 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
360 if priority == None:
361 bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
362 continue
363 try:
364 cre = re.compile(regex)
365 except re.error:
366 bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
367 continue
368 try:
369 pri = int(priority)
370 self.status.bbfile_config_priorities.append((cre, pri))
371 except ValueError:
372 bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
373
374
375 def cook(self, configuration):
376 """
377 We are building stuff here. We do the building
378 from here. By default we try to execute task
379 build.
380 """
381
382 self.configuration = configuration
383
384 if self.configuration.verbose:
385 bb.msg.set_verbose(True)
386
387 if self.configuration.debug:
388 bb.msg.set_debug_level(self.configuration.debug)
389 else:
390 bb.msg.set_debug_level(0)
391
392 if self.configuration.debug_domains:
393 bb.msg.set_debug_domains(self.configuration.debug_domains)
394
395 self.configuration.data = bb.data.init()
396
397 for f in self.configuration.file:
398 self.parseConfigurationFile( f )
399
400 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
401
402 if not self.configuration.cmd:
403 self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build"
404
405 #
406 # Special updated configuration we use for firing events
407 #
408 self.configuration.event_data = bb.data.createCopy(self.configuration.data)
409 bb.data.update_data(self.configuration.event_data)
410
411 if self.configuration.show_environment:
412 self.showEnvironment()
413 sys.exit( 0 )
414
415 # inject custom variables
416 if not bb.data.getVar("BUILDNAME", self.configuration.data):
417 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
418 bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
419
420 buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
421
422 if self.configuration.interactive:
423 self.interactiveMode()
424
425 if self.configuration.buildfile is not None:
426 bf = os.path.abspath( self.configuration.buildfile )
427 try:
428 os.stat(bf)
429 except OSError:
430 (filelist, masked) = self.collect_bbfiles()
431 regexp = re.compile(self.configuration.buildfile)
432 matches = []
433 for f in filelist:
434 if regexp.search(f) and os.path.isfile(f):
435 bf = f
436 matches.append(f)
437 if len(matches) != 1:
438 bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (self.configuration.buildfile, len(matches)))
439 for f in matches:
440 bb.msg.error(bb.msg.domain.Parsing, " %s" % f)
441 sys.exit(1)
442 bf = matches[0]
443
444 bbfile_data = bb.parse.handle(bf, self.configuration.data)
445
446 # Remove stamp for target if force mode active
447 if self.configuration.force:
448 bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, bf))
449 bb.build.del_stamp('do_%s' % self.configuration.cmd, bbfile_data)
450
451 item = bb.data.getVar('PN', bbfile_data, 1)
452 try:
453 self.tryBuildPackage(bf, item, self.configuration.cmd, bbfile_data, True)
454 except bb.build.EventException:
455 bb.msg.error(bb.msg.domain.Build, "Build of '%s' failed" % item )
456
457 sys.exit( self.stats.show() )
458
459 # initialise the parsing status now we know we will need deps
460 self.status = bb.cache.CacheData()
461
462 ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
463 self.status.ignored_dependencies = Set( ignore.split() )
464
465 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
466
467 pkgs_to_build = self.configuration.pkgs_to_build
468
469 bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
470 if bbpkgs:
471 pkgs_to_build.extend(bbpkgs.split())
472 if len(pkgs_to_build) == 0 and not self.configuration.show_versions \
473 and not self.configuration.show_environment:
474 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
475 print "for usage information."
476 sys.exit(0)
477
478 # Import Psyco if available and not disabled
479 if not self.configuration.disable_psyco:
480 try:
481 import psyco
482 except ImportError:
483 bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
484 else:
485 psyco.bind( self.parse_bbfiles )
486 else:
487 bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
488
489 try:
490 bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
491 (filelist, masked) = self.collect_bbfiles()
492 self.parse_bbfiles(filelist, masked, self.myProgressCallback)
493 bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
494 print
495 if self.configuration.parse_only:
496 bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only. Exiting.")
497 return
498
499
500 self.buildDepgraph()
501
502 if self.configuration.show_versions:
503 self.showVersions()
504 sys.exit( 0 )
505 if 'world' in pkgs_to_build:
506 self.buildWorldTargetList()
507 pkgs_to_build.remove('world')
508 for t in self.status.world_target:
509 pkgs_to_build.append(t)
510
511 if self.configuration.dot_graph:
512 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
513 sys.exit( 0 )
514
515 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.event_data))
516
517 localdata = data.createCopy(self.configuration.data)
518 bb.data.update_data(localdata)
519 bb.data.expandKeys(localdata)
520
521 taskdata = bb.taskdata.TaskData(self.configuration.abort)
522
523 runlist = []
524 try:
525 for k in pkgs_to_build:
526 taskdata.add_provider(localdata, self.status, k)
527 runlist.append([k, "do_%s" % self.configuration.cmd])
528 taskdata.add_unresolved(localdata, self.status)
529 except bb.providers.NoProvider:
530 sys.exit(1)
531
532 rq = bb.runqueue.RunQueue()
533 rq.prepare_runqueue(self, self.configuration.data, self.status, taskdata, runlist)
534 try:
535 failures = rq.execute_runqueue(self, self.configuration.data, self.status, taskdata, runlist)
536 except runqueue.TaskFailure, fnids:
537 for fnid in fnids:
538 bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
539 sys.exit(1)
540 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.event_data, failures))
541
542 sys.exit( self.stats.show() )
543
544 except KeyboardInterrupt:
545 bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
546 sys.exit(1)
547
548 def get_bbfiles( self, path = os.getcwd() ):
549 """Get list of default .bb files by reading out the current directory"""
550 contents = os.listdir(path)
551 bbfiles = []
552 for f in contents:
553 (root, ext) = os.path.splitext(f)
554 if ext == ".bb":
555 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
556 return bbfiles
557
558 def find_bbfiles( self, path ):
559 """Find all the .bb files in a directory (uses find)"""
560 findcmd = 'find ' + path + ' -name *.bb | grep -v SCCS/'
561 try:
562 finddata = os.popen(findcmd)
563 except OSError:
564 return []
565 return finddata.readlines()
566
567 def collect_bbfiles( self ):
568 """Collect all available .bb build files"""
569 parsed, cached, skipped, masked = 0, 0, 0, 0
570 self.bb_cache = bb.cache.init(self)
571
572 files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
573 data.setVar("BBFILES", " ".join(files), self.configuration.data)
574
575 if not len(files):
576 files = self.get_bbfiles()
577
578 if not len(files):
579 bb.msg.error(bb.msg.domain.Collection, "no files to build.")
580
581 newfiles = []
582 for f in files:
583 if os.path.isdir(f):
584 dirfiles = self.find_bbfiles(f)
585 if dirfiles:
586 newfiles += dirfiles
587 continue
588 newfiles += glob.glob(f) or [ f ]
589
590 bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
591
592 if not bbmask:
593 return (newfiles, 0)
594
595 try:
596 bbmask_compiled = re.compile(bbmask)
597 except sre_constants.error:
598 bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
599
600 finalfiles = []
601 for i in xrange( len( newfiles ) ):
602 f = newfiles[i]
603 if bbmask and bbmask_compiled.search(f):
604 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
605 masked += 1
606 continue
607 finalfiles.append(f)
608
609 return (finalfiles, masked)
610
611 def parse_bbfiles(self, filelist, masked, progressCallback = None):
612 parsed, cached, skipped = 0, 0, 0
613 for i in xrange( len( filelist ) ):
614 f = filelist[i]
615
616 bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
617
618 # read a file's metadata
619 try:
620 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data)
621 if skip:
622 skipped += 1
623 bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
624 self.bb_cache.skip(f)
625 continue
626 elif fromCache: cached += 1
627 else: parsed += 1
628 deps = None
629
630 # Disabled by RP as was no longer functional
631 # allow metadata files to add items to BBFILES
632 #data.update_data(self.pkgdata[f])
633 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
634 #if addbbfiles:
635 # for aof in addbbfiles.split():
636 # if not files.count(aof):
637 # if not os.path.isabs(aof):
638 # aof = os.path.join(os.path.dirname(f),aof)
639 # files.append(aof)
640
641 self.bb_cache.handle_data(f, self.status)
642
643 # now inform the caller
644 if progressCallback is not None:
645 progressCallback( i + 1, len( filelist ), f, fromCache )
646
647 except IOError, e:
648 self.bb_cache.remove(f)
649 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
650 pass
651 except KeyboardInterrupt:
652 self.bb_cache.sync()
653 raise
654 except Exception, e:
655 self.bb_cache.remove(f)
656 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
657 except:
658 self.bb_cache.remove(f)
659 raise
660
661 if progressCallback is not None:
662 print "\r" # need newline after Handling Bitbake files message
663 bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
664
665 self.bb_cache.sync()