summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/taskdata.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/taskdata.py')
-rw-r--r--bitbake/lib/bb/taskdata.py651
1 files changed, 651 insertions, 0 deletions
diff --git a/bitbake/lib/bb/taskdata.py b/bitbake/lib/bb/taskdata.py
new file mode 100644
index 0000000..af72a1f
--- /dev/null
+++ b/bitbake/lib/bb/taskdata.py
@@ -0,0 +1,651 @@
1#!/usr/bin/env python
2# ex:ts=4:sw=4:sts=4:et
3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4"""
5BitBake 'TaskData' implementation
6
7Task data collection and handling
8
9"""
10
11# Copyright (C) 2006 Richard Purdie
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License version 2 as
15# published by the Free Software Foundation.
16#
17# This program is distributed in the hope that it will be useful,
18# but WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20# GNU General Public License for more details.
21#
22# You should have received a copy of the GNU General Public License along
23# with this program; if not, write to the Free Software Foundation, Inc.,
24# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25
26import logging
27import re
28import bb
29
30logger = logging.getLogger("BitBake.TaskData")
31
32def re_match_strings(target, strings):
33 """
34 Whether or not the string 'target' matches
35 any one string of the strings which can be regular expression string
36 """
37 return any(name == target or re.match(name, target)
38 for name in strings)
39
40class TaskData:
41 """
42 BitBake Task Data implementation
43 """
44 def __init__(self, abort = True, tryaltconfigs = False, skiplist = None):
45 self.build_names_index = []
46 self.run_names_index = []
47 self.fn_index = []
48
49 self.build_targets = {}
50 self.run_targets = {}
51
52 self.external_targets = []
53
54 self.tasks_fnid = []
55 self.tasks_name = []
56 self.tasks_tdepends = []
57 self.tasks_idepends = []
58 self.tasks_irdepends = []
59 # Cache to speed up task ID lookups
60 self.tasks_lookup = {}
61
62 self.depids = {}
63 self.rdepids = {}
64
65 self.consider_msgs_cache = []
66
67 self.failed_deps = []
68 self.failed_rdeps = []
69 self.failed_fnids = []
70
71 self.abort = abort
72 self.tryaltconfigs = tryaltconfigs
73
74 self.skiplist = skiplist
75
76 def getbuild_id(self, name):
77 """
78 Return an ID number for the build target name.
79 If it doesn't exist, create one.
80 """
81 if not name in self.build_names_index:
82 self.build_names_index.append(name)
83 return len(self.build_names_index) - 1
84
85 return self.build_names_index.index(name)
86
87 def getrun_id(self, name):
88 """
89 Return an ID number for the run target name.
90 If it doesn't exist, create one.
91 """
92 if not name in self.run_names_index:
93 self.run_names_index.append(name)
94 return len(self.run_names_index) - 1
95
96 return self.run_names_index.index(name)
97
98 def getfn_id(self, name):
99 """
100 Return an ID number for the filename.
101 If it doesn't exist, create one.
102 """
103 if not name in self.fn_index:
104 self.fn_index.append(name)
105 return len(self.fn_index) - 1
106
107 return self.fn_index.index(name)
108
109 def gettask_ids(self, fnid):
110 """
111 Return an array of the ID numbers matching a given fnid.
112 """
113 ids = []
114 if fnid in self.tasks_lookup:
115 for task in self.tasks_lookup[fnid]:
116 ids.append(self.tasks_lookup[fnid][task])
117 return ids
118
119 def gettask_id_fromfnid(self, fnid, task):
120 """
121 Return an ID number for the task matching fnid and task.
122 """
123 if fnid in self.tasks_lookup:
124 if task in self.tasks_lookup[fnid]:
125 return self.tasks_lookup[fnid][task]
126
127 return None
128
129 def gettask_id(self, fn, task, create = True):
130 """
131 Return an ID number for the task matching fn and task.
132 If it doesn't exist, create one by default.
133 Optionally return None instead.
134 """
135 fnid = self.getfn_id(fn)
136
137 if fnid in self.tasks_lookup:
138 if task in self.tasks_lookup[fnid]:
139 return self.tasks_lookup[fnid][task]
140
141 if not create:
142 return None
143
144 self.tasks_name.append(task)
145 self.tasks_fnid.append(fnid)
146 self.tasks_tdepends.append([])
147 self.tasks_idepends.append([])
148 self.tasks_irdepends.append([])
149
150 listid = len(self.tasks_name) - 1
151
152 if fnid not in self.tasks_lookup:
153 self.tasks_lookup[fnid] = {}
154 self.tasks_lookup[fnid][task] = listid
155
156 return listid
157
158 def add_tasks(self, fn, dataCache):
159 """
160 Add tasks for a given fn to the database
161 """
162
163 task_deps = dataCache.task_deps[fn]
164
165 fnid = self.getfn_id(fn)
166
167 if fnid in self.failed_fnids:
168 bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...")
169
170 # Check if we've already seen this fn
171 if fnid in self.tasks_fnid:
172 return
173
174 for task in task_deps['tasks']:
175
176 # Work out task dependencies
177 parentids = []
178 for dep in task_deps['parents'][task]:
179 if dep not in task_deps['tasks']:
180 bb.debug(2, "Not adding dependeny of %s on %s since %s does not exist" % (task, dep, dep))
181 continue
182 parentid = self.gettask_id(fn, dep)
183 parentids.append(parentid)
184 taskid = self.gettask_id(fn, task)
185 self.tasks_tdepends[taskid].extend(parentids)
186
187 # Touch all intertask dependencies
188 if 'depends' in task_deps and task in task_deps['depends']:
189 ids = []
190 for dep in task_deps['depends'][task].split():
191 if dep:
192 if ":" not in dep:
193 bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'depends' should be specified in the form 'packagename:task'" % (fn, dep))
194 ids.append(((self.getbuild_id(dep.split(":")[0])), dep.split(":")[1]))
195 self.tasks_idepends[taskid].extend(ids)
196 if 'rdepends' in task_deps and task in task_deps['rdepends']:
197 ids = []
198 for dep in task_deps['rdepends'][task].split():
199 if dep:
200 if ":" not in dep:
201 bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'rdepends' should be specified in the form 'packagename:task'" % (fn, dep))
202 ids.append(((self.getrun_id(dep.split(":")[0])), dep.split(":")[1]))
203 self.tasks_irdepends[taskid].extend(ids)
204
205
206 # Work out build dependencies
207 if not fnid in self.depids:
208 dependids = {}
209 for depend in dataCache.deps[fn]:
210 dependids[self.getbuild_id(depend)] = None
211 self.depids[fnid] = dependids.keys()
212 logger.debug(2, "Added dependencies %s for %s", str(dataCache.deps[fn]), fn)
213
214 # Work out runtime dependencies
215 if not fnid in self.rdepids:
216 rdependids = {}
217 rdepends = dataCache.rundeps[fn]
218 rrecs = dataCache.runrecs[fn]
219 rdependlist = []
220 rreclist = []
221 for package in rdepends:
222 for rdepend in rdepends[package]:
223 rdependlist.append(rdepend)
224 rdependids[self.getrun_id(rdepend)] = None
225 for package in rrecs:
226 for rdepend in rrecs[package]:
227 rreclist.append(rdepend)
228 rdependids[self.getrun_id(rdepend)] = None
229 if rdependlist:
230 logger.debug(2, "Added runtime dependencies %s for %s", str(rdependlist), fn)
231 if rreclist:
232 logger.debug(2, "Added runtime recommendations %s for %s", str(rreclist), fn)
233 self.rdepids[fnid] = rdependids.keys()
234
235 for dep in self.depids[fnid]:
236 if dep in self.failed_deps:
237 self.fail_fnid(fnid)
238 return
239 for dep in self.rdepids[fnid]:
240 if dep in self.failed_rdeps:
241 self.fail_fnid(fnid)
242 return
243
244 def have_build_target(self, target):
245 """
246 Have we a build target matching this name?
247 """
248 targetid = self.getbuild_id(target)
249
250 if targetid in self.build_targets:
251 return True
252 return False
253
254 def have_runtime_target(self, target):
255 """
256 Have we a runtime target matching this name?
257 """
258 targetid = self.getrun_id(target)
259
260 if targetid in self.run_targets:
261 return True
262 return False
263
264 def add_build_target(self, fn, item):
265 """
266 Add a build target.
267 If already present, append the provider fn to the list
268 """
269 targetid = self.getbuild_id(item)
270 fnid = self.getfn_id(fn)
271
272 if targetid in self.build_targets:
273 if fnid in self.build_targets[targetid]:
274 return
275 self.build_targets[targetid].append(fnid)
276 return
277 self.build_targets[targetid] = [fnid]
278
279 def add_runtime_target(self, fn, item):
280 """
281 Add a runtime target.
282 If already present, append the provider fn to the list
283 """
284 targetid = self.getrun_id(item)
285 fnid = self.getfn_id(fn)
286
287 if targetid in self.run_targets:
288 if fnid in self.run_targets[targetid]:
289 return
290 self.run_targets[targetid].append(fnid)
291 return
292 self.run_targets[targetid] = [fnid]
293
294 def mark_external_target(self, item):
295 """
296 Mark a build target as being externally requested
297 """
298 targetid = self.getbuild_id(item)
299
300 if targetid not in self.external_targets:
301 self.external_targets.append(targetid)
302
303 def get_unresolved_build_targets(self, dataCache):
304 """
305 Return a list of build targets who's providers
306 are unknown.
307 """
308 unresolved = []
309 for target in self.build_names_index:
310 if re_match_strings(target, dataCache.ignored_dependencies):
311 continue
312 if self.build_names_index.index(target) in self.failed_deps:
313 continue
314 if not self.have_build_target(target):
315 unresolved.append(target)
316 return unresolved
317
318 def get_unresolved_run_targets(self, dataCache):
319 """
320 Return a list of runtime targets who's providers
321 are unknown.
322 """
323 unresolved = []
324 for target in self.run_names_index:
325 if re_match_strings(target, dataCache.ignored_dependencies):
326 continue
327 if self.run_names_index.index(target) in self.failed_rdeps:
328 continue
329 if not self.have_runtime_target(target):
330 unresolved.append(target)
331 return unresolved
332
333 def get_provider(self, item):
334 """
335 Return a list of providers of item
336 """
337 targetid = self.getbuild_id(item)
338
339 return self.build_targets[targetid]
340
341 def get_dependees(self, itemid):
342 """
343 Return a list of targets which depend on item
344 """
345 dependees = []
346 for fnid in self.depids:
347 if itemid in self.depids[fnid]:
348 dependees.append(fnid)
349 return dependees
350
351 def get_dependees_str(self, item):
352 """
353 Return a list of targets which depend on item as a user readable string
354 """
355 itemid = self.getbuild_id(item)
356 dependees = []
357 for fnid in self.depids:
358 if itemid in self.depids[fnid]:
359 dependees.append(self.fn_index[fnid])
360 return dependees
361
362 def get_rdependees(self, itemid):
363 """
364 Return a list of targets which depend on runtime item
365 """
366 dependees = []
367 for fnid in self.rdepids:
368 if itemid in self.rdepids[fnid]:
369 dependees.append(fnid)
370 return dependees
371
372 def get_rdependees_str(self, item):
373 """
374 Return a list of targets which depend on runtime item as a user readable string
375 """
376 itemid = self.getrun_id(item)
377 dependees = []
378 for fnid in self.rdepids:
379 if itemid in self.rdepids[fnid]:
380 dependees.append(self.fn_index[fnid])
381 return dependees
382
383 def get_reasons(self, item, runtime=False):
384 """
385 Get the reason(s) for an item not being provided, if any
386 """
387 reasons = []
388 if self.skiplist:
389 for fn in self.skiplist:
390 skipitem = self.skiplist[fn]
391 if skipitem.pn == item:
392 reasons.append("%s was skipped: %s" % (skipitem.pn, skipitem.skipreason))
393 elif runtime and item in skipitem.rprovides:
394 reasons.append("%s RPROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
395 elif not runtime and item in skipitem.provides:
396 reasons.append("%s PROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
397 return reasons
398
399 def get_close_matches(self, item, provider_list):
400 import difflib
401 if self.skiplist:
402 skipped = []
403 for fn in self.skiplist:
404 skipped.append(self.skiplist[fn].pn)
405 full_list = provider_list + skipped
406 else:
407 full_list = provider_list
408 return difflib.get_close_matches(item, full_list, cutoff=0.7)
409
410 def add_provider(self, cfgData, dataCache, item):
411 try:
412 self.add_provider_internal(cfgData, dataCache, item)
413 except bb.providers.NoProvider:
414 if self.abort:
415 raise
416 self.remove_buildtarget(self.getbuild_id(item))
417
418 self.mark_external_target(item)
419
420 def add_provider_internal(self, cfgData, dataCache, item):
421 """
422 Add the providers of item to the task data
423 Mark entries were specifically added externally as against dependencies
424 added internally during dependency resolution
425 """
426
427 if re_match_strings(item, dataCache.ignored_dependencies):
428 return
429
430 if not item in dataCache.providers:
431 bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item), reasons=self.get_reasons(item), close_matches=self.get_close_matches(item, dataCache.providers.keys())), cfgData)
432 raise bb.providers.NoProvider(item)
433
434 if self.have_build_target(item):
435 return
436
437 all_p = dataCache.providers[item]
438
439 eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
440 eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids]
441
442 if not eligible:
443 bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData)
444 raise bb.providers.NoProvider(item)
445
446 if len(eligible) > 1 and foundUnique == False:
447 if item not in self.consider_msgs_cache:
448 providers_list = []
449 for fn in eligible:
450 providers_list.append(dataCache.pkg_fn[fn])
451 bb.event.fire(bb.event.MultipleProviders(item, providers_list), cfgData)
452 self.consider_msgs_cache.append(item)
453
454 for fn in eligible:
455 fnid = self.getfn_id(fn)
456 if fnid in self.failed_fnids:
457 continue
458 logger.debug(2, "adding %s to satisfy %s", fn, item)
459 self.add_build_target(fn, item)
460 self.add_tasks(fn, dataCache)
461
462
463 #item = dataCache.pkg_fn[fn]
464
465 def add_rprovider(self, cfgData, dataCache, item):
466 """
467 Add the runtime providers of item to the task data
468 (takes item names from RDEPENDS/PACKAGES namespace)
469 """
470
471 if re_match_strings(item, dataCache.ignored_dependencies):
472 return
473
474 if self.have_runtime_target(item):
475 return
476
477 all_p = bb.providers.getRuntimeProviders(dataCache, item)
478
479 if not all_p:
480 bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item), reasons=self.get_reasons(item, True)), cfgData)
481 raise bb.providers.NoRProvider(item)
482
483 eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache)
484 eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids]
485
486 if not eligible:
487 bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData)
488 raise bb.providers.NoRProvider(item)
489
490 if len(eligible) > 1 and numberPreferred == 0:
491 if item not in self.consider_msgs_cache:
492 providers_list = []
493 for fn in eligible:
494 providers_list.append(dataCache.pkg_fn[fn])
495 bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
496 self.consider_msgs_cache.append(item)
497
498 if numberPreferred > 1:
499 if item not in self.consider_msgs_cache:
500 providers_list = []
501 for fn in eligible:
502 providers_list.append(dataCache.pkg_fn[fn])
503 bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
504 self.consider_msgs_cache.append(item)
505 raise bb.providers.MultipleRProvider(item)
506
507 # run through the list until we find one that we can build
508 for fn in eligible:
509 fnid = self.getfn_id(fn)
510 if fnid in self.failed_fnids:
511 continue
512 logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item)
513 self.add_runtime_target(fn, item)
514 self.add_tasks(fn, dataCache)
515
516 def fail_fnid(self, fnid, missing_list = []):
517 """
518 Mark a file as failed (unbuildable)
519 Remove any references from build and runtime provider lists
520
521 missing_list, A list of missing requirements for this target
522 """
523 if fnid in self.failed_fnids:
524 return
525 logger.debug(1, "File '%s' is unbuildable, removing...", self.fn_index[fnid])
526 self.failed_fnids.append(fnid)
527 for target in self.build_targets:
528 if fnid in self.build_targets[target]:
529 self.build_targets[target].remove(fnid)
530 if len(self.build_targets[target]) == 0:
531 self.remove_buildtarget(target, missing_list)
532 for target in self.run_targets:
533 if fnid in self.run_targets[target]:
534 self.run_targets[target].remove(fnid)
535 if len(self.run_targets[target]) == 0:
536 self.remove_runtarget(target, missing_list)
537
538 def remove_buildtarget(self, targetid, missing_list = []):
539 """
540 Mark a build target as failed (unbuildable)
541 Trigger removal of any files that have this as a dependency
542 """
543 if not missing_list:
544 missing_list = [self.build_names_index[targetid]]
545 else:
546 missing_list = [self.build_names_index[targetid]] + missing_list
547 logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", self.build_names_index[targetid], missing_list)
548 self.failed_deps.append(targetid)
549 dependees = self.get_dependees(targetid)
550 for fnid in dependees:
551 self.fail_fnid(fnid, missing_list)
552 for taskid in xrange(len(self.tasks_idepends)):
553 idepends = self.tasks_idepends[taskid]
554 for (idependid, idependtask) in idepends:
555 if idependid == targetid:
556 self.fail_fnid(self.tasks_fnid[taskid], missing_list)
557
558 if self.abort and targetid in self.external_targets:
559 target = self.build_names_index[targetid]
560 logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
561 raise bb.providers.NoProvider(target)
562
563 def remove_runtarget(self, targetid, missing_list = []):
564 """
565 Mark a run target as failed (unbuildable)
566 Trigger removal of any files that have this as a dependency
567 """
568 if not missing_list:
569 missing_list = [self.run_names_index[targetid]]
570 else:
571 missing_list = [self.run_names_index[targetid]] + missing_list
572
573 logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", self.run_names_index[targetid], missing_list)
574 self.failed_rdeps.append(targetid)
575 dependees = self.get_rdependees(targetid)
576 for fnid in dependees:
577 self.fail_fnid(fnid, missing_list)
578 for taskid in xrange(len(self.tasks_irdepends)):
579 irdepends = self.tasks_irdepends[taskid]
580 for (idependid, idependtask) in irdepends:
581 if idependid == targetid:
582 self.fail_fnid(self.tasks_fnid[taskid], missing_list)
583
584 def add_unresolved(self, cfgData, dataCache):
585 """
586 Resolve all unresolved build and runtime targets
587 """
588 logger.info("Resolving any missing task queue dependencies")
589 while True:
590 added = 0
591 for target in self.get_unresolved_build_targets(dataCache):
592 try:
593 self.add_provider_internal(cfgData, dataCache, target)
594 added = added + 1
595 except bb.providers.NoProvider:
596 targetid = self.getbuild_id(target)
597 if self.abort and targetid in self.external_targets:
598 raise
599 self.remove_buildtarget(targetid)
600 for target in self.get_unresolved_run_targets(dataCache):
601 try:
602 self.add_rprovider(cfgData, dataCache, target)
603 added = added + 1
604 except (bb.providers.NoRProvider, bb.providers.MultipleRProvider):
605 self.remove_runtarget(self.getrun_id(target))
606 logger.debug(1, "Resolved " + str(added) + " extra dependencies")
607 if added == 0:
608 break
609 # self.dump_data()
610
611 def dump_data(self):
612 """
613 Dump some debug information on the internal data structures
614 """
615 logger.debug(3, "build_names:")
616 logger.debug(3, ", ".join(self.build_names_index))
617
618 logger.debug(3, "run_names:")
619 logger.debug(3, ", ".join(self.run_names_index))
620
621 logger.debug(3, "build_targets:")
622 for buildid in xrange(len(self.build_names_index)):
623 target = self.build_names_index[buildid]
624 targets = "None"
625 if buildid in self.build_targets:
626 targets = self.build_targets[buildid]
627 logger.debug(3, " (%s)%s: %s", buildid, target, targets)
628
629 logger.debug(3, "run_targets:")
630 for runid in xrange(len(self.run_names_index)):
631 target = self.run_names_index[runid]
632 targets = "None"
633 if runid in self.run_targets:
634 targets = self.run_targets[runid]
635 logger.debug(3, " (%s)%s: %s", runid, target, targets)
636
637 logger.debug(3, "tasks:")
638 for task in xrange(len(self.tasks_name)):
639 logger.debug(3, " (%s)%s - %s: %s",
640 task,
641 self.fn_index[self.tasks_fnid[task]],
642 self.tasks_name[task],
643 self.tasks_tdepends[task])
644
645 logger.debug(3, "dependency ids (per fn):")
646 for fnid in self.depids:
647 logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.depids[fnid])
648
649 logger.debug(3, "runtime dependency ids (per fn):")
650 for fnid in self.rdepids:
651 logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.rdepids[fnid])