summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/taskdata.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/taskdata.py')
-rw-r--r--bitbake/lib/bb/taskdata.py586
1 files changed, 0 insertions, 586 deletions
diff --git a/bitbake/lib/bb/taskdata.py b/bitbake/lib/bb/taskdata.py
deleted file mode 100644
index 66545a65af..0000000000
--- a/bitbake/lib/bb/taskdata.py
+++ /dev/null
@@ -1,586 +0,0 @@
1"""
2BitBake 'TaskData' implementation
3
4Task data collection and handling
5
6"""
7
8# Copyright (C) 2006 Richard Purdie
9#
10# SPDX-License-Identifier: GPL-2.0-only
11#
12
13import logging
14import re
15import bb
16
17logger = logging.getLogger("BitBake.TaskData")
18
19def re_match_strings(target, strings):
20 """
21 Whether or not the string 'target' matches
22 any one string of the strings which can be regular expression string
23 """
24 for name in strings:
25 if name.startswith("^") or name.endswith("$"):
26 if re.match(name, target):
27 return True
28 elif name == target:
29 return True
30 return False
31
32class TaskEntry:
33 def __init__(self):
34 self.tdepends = []
35 self.idepends = []
36 self.irdepends = []
37
38class TaskData:
39 """
40 BitBake Task Data implementation
41 """
42 def __init__(self, halt = True, skiplist = None, allowincomplete = False):
43 self.build_targets = {}
44 self.run_targets = {}
45
46 self.external_targets = []
47
48 self.seenfns = []
49 self.taskentries = {}
50
51 self.depids = {}
52 self.rdepids = {}
53
54 self.consider_msgs_cache = []
55
56 self.failed_deps = []
57 self.failed_rdeps = []
58 self.failed_fns = []
59
60 self.halt = halt
61 self.allowincomplete = allowincomplete
62
63 self.skiplist = skiplist
64
65 self.mcdepends = []
66
67 def add_tasks(self, fn, dataCache):
68 """
69 Add tasks for a given fn to the database
70 """
71
72 task_deps = dataCache.task_deps[fn]
73
74 if fn in self.failed_fns:
75 bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...")
76
77 # Check if we've already seen this fn
78 if fn in self.seenfns:
79 return
80
81 self.seenfns.append(fn)
82
83 self.add_extra_deps(fn, dataCache)
84
85 def add_mcdepends(task):
86 for dep in task_deps['mcdepends'][task].split():
87 if len(dep.split(':')) != 5:
88 bb.msg.fatal("TaskData", "Error for %s:%s[%s], multiconfig dependency %s does not contain exactly four ':' characters.\n Task '%s' should be specified in the form 'mc:fromMC:toMC:packagename:task'" % (fn, task, 'mcdepends', dep, 'mcdepends'))
89 if dep not in self.mcdepends:
90 self.mcdepends.append(dep)
91
92 # Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends
93 def handle_deps(task, dep_name, depends, seen):
94 if dep_name in task_deps and task in task_deps[dep_name]:
95 ids = []
96 for dep in task_deps[dep_name][task].split():
97 if dep:
98 parts = dep.split(":")
99 if len(parts) != 2:
100 bb.msg.fatal("TaskData", "Error for %s:%s[%s], dependency %s in '%s' does not contain exactly one ':' character.\n Task '%s' should be specified in the form 'packagename:task'" % (fn, task, dep_name, dep, task_deps[dep_name][task], dep_name))
101 ids.append((parts[0], parts[1]))
102 seen(parts[0])
103 depends.extend(ids)
104
105 for task in task_deps['tasks']:
106
107 tid = "%s:%s" % (fn, task)
108 self.taskentries[tid] = TaskEntry()
109
110 # Work out task dependencies
111 parentids = []
112 for dep in task_deps['parents'][task]:
113 if dep not in task_deps['tasks']:
114 bb.debug(2, "Not adding dependency of %s on %s since %s does not exist" % (task, dep, dep))
115 continue
116 parentid = "%s:%s" % (fn, dep)
117 parentids.append(parentid)
118 self.taskentries[tid].tdepends.extend(parentids)
119
120
121 # Touch all intertask dependencies
122 handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target)
123 handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target)
124
125 if 'mcdepends' in task_deps and task in task_deps['mcdepends']:
126 add_mcdepends(task)
127
128 # Work out build dependencies
129 if not fn in self.depids:
130 dependids = set()
131 for depend in dataCache.deps[fn]:
132 dependids.add(depend)
133 self.depids[fn] = list(dependids)
134 logger.debug2("Added dependencies %s for %s", str(dataCache.deps[fn]), fn)
135
136 # Work out runtime dependencies
137 if not fn in self.rdepids:
138 rdependids = set()
139 rdepends = dataCache.rundeps[fn]
140 rrecs = dataCache.runrecs[fn]
141 rdependlist = []
142 rreclist = []
143 for package in rdepends:
144 for rdepend in rdepends[package]:
145 rdependlist.append(rdepend)
146 rdependids.add(rdepend)
147 for package in rrecs:
148 for rdepend in rrecs[package]:
149 rreclist.append(rdepend)
150 rdependids.add(rdepend)
151 if rdependlist:
152 logger.debug2("Added runtime dependencies %s for %s", str(rdependlist), fn)
153 if rreclist:
154 logger.debug2("Added runtime recommendations %s for %s", str(rreclist), fn)
155 self.rdepids[fn] = list(rdependids)
156
157 for dep in self.depids[fn]:
158 self.seen_build_target(dep)
159 if dep in self.failed_deps:
160 self.fail_fn(fn)
161 return
162 for dep in self.rdepids[fn]:
163 self.seen_run_target(dep)
164 if dep in self.failed_rdeps:
165 self.fail_fn(fn)
166 return
167
168 def add_extra_deps(self, fn, dataCache):
169 func = dataCache.extradepsfunc.get(fn, None)
170 if func:
171 bb.providers.buildWorldTargetList(dataCache)
172 pn = dataCache.pkg_fn[fn]
173 params = {'deps': dataCache.deps[fn],
174 'world_target': dataCache.world_target,
175 'pkg_pn': dataCache.pkg_pn,
176 'self_pn': pn}
177 funcname = '_%s_calculate_extra_depends' % pn.replace('-', '_')
178 paramlist = ','.join(params.keys())
179 func = 'def %s(%s):\n%s\n\n%s(%s)' % (funcname, paramlist, func, funcname, paramlist)
180 bb.utils.better_exec(func, params)
181
182
183 def have_build_target(self, target):
184 """
185 Have we a build target matching this name?
186 """
187 if target in self.build_targets and self.build_targets[target]:
188 return True
189 return False
190
191 def have_runtime_target(self, target):
192 """
193 Have we a runtime target matching this name?
194 """
195 if target in self.run_targets and self.run_targets[target]:
196 return True
197 return False
198
199 def seen_build_target(self, name):
200 """
201 Maintain a list of build targets
202 """
203 if name not in self.build_targets:
204 self.build_targets[name] = []
205
206 def add_build_target(self, fn, item):
207 """
208 Add a build target.
209 If already present, append the provider fn to the list
210 """
211 if item in self.build_targets:
212 if fn in self.build_targets[item]:
213 return
214 self.build_targets[item].append(fn)
215 return
216 self.build_targets[item] = [fn]
217
218 def seen_run_target(self, name):
219 """
220 Maintain a list of runtime build targets
221 """
222 if name not in self.run_targets:
223 self.run_targets[name] = []
224
225 def add_runtime_target(self, fn, item):
226 """
227 Add a runtime target.
228 If already present, append the provider fn to the list
229 """
230 if item in self.run_targets:
231 if fn in self.run_targets[item]:
232 return
233 self.run_targets[item].append(fn)
234 return
235 self.run_targets[item] = [fn]
236
237 def mark_external_target(self, target):
238 """
239 Mark a build target as being externally requested
240 """
241 if target not in self.external_targets:
242 self.external_targets.append(target)
243
244 def get_unresolved_build_targets(self, dataCache):
245 """
246 Return a list of build targets who's providers
247 are unknown.
248 """
249 unresolved = []
250 for target in self.build_targets:
251 if re_match_strings(target, dataCache.ignored_dependencies):
252 continue
253 if target in self.failed_deps:
254 continue
255 if not self.build_targets[target]:
256 unresolved.append(target)
257 return unresolved
258
259 def get_unresolved_run_targets(self, dataCache):
260 """
261 Return a list of runtime targets who's providers
262 are unknown.
263 """
264 unresolved = []
265 for target in self.run_targets:
266 if re_match_strings(target, dataCache.ignored_dependencies):
267 continue
268 if target in self.failed_rdeps:
269 continue
270 if not self.run_targets[target]:
271 unresolved.append(target)
272 return unresolved
273
274 def get_provider(self, item):
275 """
276 Return a list of providers of item
277 """
278 return self.build_targets[item]
279
280 def get_dependees(self, item):
281 """
282 Return a list of targets which depend on item
283 """
284 dependees = []
285 for fn in self.depids:
286 if item in self.depids[fn]:
287 dependees.append(fn)
288 return dependees
289
290 def get_rdependees(self, item):
291 """
292 Return a list of targets which depend on runtime item
293 """
294 dependees = []
295 for fn in self.rdepids:
296 if item in self.rdepids[fn]:
297 dependees.append(fn)
298 return dependees
299
300 def get_reasons(self, item, runtime=False):
301 """
302 Get the reason(s) for an item not being provided, if any
303 """
304 reasons = []
305 if self.skiplist:
306 for fn in self.skiplist:
307 skipitem = self.skiplist[fn]
308 if skipitem.pn == item:
309 reasons.append("%s was skipped: %s" % (skipitem.pn, skipitem.skipreason))
310 elif runtime and item in skipitem.rprovides:
311 reasons.append("%s RPROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
312 elif not runtime and item in skipitem.provides:
313 reasons.append("%s PROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
314 return reasons
315
316 def get_close_matches(self, item, provider_list):
317 import difflib
318 if self.skiplist:
319 skipped = []
320 for fn in self.skiplist:
321 skipped.append(self.skiplist[fn].pn)
322 full_list = provider_list + skipped
323 else:
324 full_list = provider_list
325 return difflib.get_close_matches(item, full_list, cutoff=0.7)
326
327 def add_provider(self, cfgData, dataCache, item):
328 try:
329 self.add_provider_internal(cfgData, dataCache, item)
330 except bb.providers.NoProvider:
331 if self.halt:
332 raise
333 self.remove_buildtarget(item)
334
335 self.mark_external_target(item)
336
337 def add_provider_internal(self, cfgData, dataCache, item):
338 """
339 Add the providers of item to the task data
340 Mark entries were specifically added externally as against dependencies
341 added internally during dependency resolution
342 """
343
344 if re_match_strings(item, dataCache.ignored_dependencies):
345 return
346
347 if not item in dataCache.providers:
348 close_matches = self.get_close_matches(item, list(dataCache.providers.keys()))
349 # Is it in RuntimeProviders ?
350 all_p = bb.providers.getRuntimeProviders(dataCache, item)
351 for fn in all_p:
352 new = dataCache.pkg_fn[fn] + " RPROVIDES " + item
353 if new not in close_matches:
354 close_matches.append(new)
355 bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData)
356 raise bb.providers.NoProvider(item)
357
358 if self.have_build_target(item):
359 return
360
361 all_p = dataCache.providers[item]
362
363 eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
364 eligible = [p for p in eligible if not p in self.failed_fns]
365
366 if not eligible:
367 bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData)
368 raise bb.providers.NoProvider(item)
369
370 if len(eligible) > 1 and not foundUnique:
371 if item not in self.consider_msgs_cache:
372 providers_list = []
373 for fn in eligible:
374 providers_list.append(dataCache.pkg_fn[fn])
375 bb.event.fire(bb.event.MultipleProviders(item, providers_list), cfgData)
376 self.consider_msgs_cache.append(item)
377
378 for fn in eligible:
379 if fn in self.failed_fns:
380 continue
381 logger.debug2("adding %s to satisfy %s", fn, item)
382 self.add_build_target(fn, item)
383 self.add_tasks(fn, dataCache)
384
385
386 #item = dataCache.pkg_fn[fn]
387
388 def add_rprovider(self, cfgData, dataCache, item):
389 """
390 Add the runtime providers of item to the task data
391 (takes item names from RDEPENDS/PACKAGES namespace)
392 """
393
394 if re_match_strings(item, dataCache.ignored_dependencies):
395 return
396
397 if self.have_runtime_target(item):
398 return
399
400 all_p = bb.providers.getRuntimeProviders(dataCache, item)
401
402 if not all_p:
403 bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=self.get_reasons(item, True)), cfgData)
404 raise bb.providers.NoRProvider(item)
405
406 eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache)
407 eligible = [p for p in eligible if not p in self.failed_fns]
408
409 if not eligible:
410 bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData)
411 raise bb.providers.NoRProvider(item)
412
413 if len(eligible) > 1 and numberPreferred == 0:
414 if item not in self.consider_msgs_cache:
415 providers_list = []
416 for fn in eligible:
417 providers_list.append(dataCache.pkg_fn[fn])
418 bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
419 self.consider_msgs_cache.append(item)
420
421 if numberPreferred > 1:
422 if item not in self.consider_msgs_cache:
423 providers_list = []
424 for fn in eligible:
425 providers_list.append(dataCache.pkg_fn[fn])
426 bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
427 self.consider_msgs_cache.append(item)
428 raise bb.providers.MultipleRProvider(item)
429
430 # run through the list until we find one that we can build
431 for fn in eligible:
432 if fn in self.failed_fns:
433 continue
434 logger.debug2("adding '%s' to satisfy runtime '%s'", fn, item)
435 self.add_runtime_target(fn, item)
436 self.add_tasks(fn, dataCache)
437
438 def fail_fn(self, fn, missing_list=None):
439 """
440 Mark a file as failed (unbuildable)
441 Remove any references from build and runtime provider lists
442
443 missing_list, A list of missing requirements for this target
444 """
445 if fn in self.failed_fns:
446 return
447 if not missing_list:
448 missing_list = []
449 logger.debug("File '%s' is unbuildable, removing...", fn)
450 self.failed_fns.append(fn)
451 for target in self.build_targets:
452 if fn in self.build_targets[target]:
453 self.build_targets[target].remove(fn)
454 if not self.build_targets[target]:
455 self.remove_buildtarget(target, missing_list)
456 for target in self.run_targets:
457 if fn in self.run_targets[target]:
458 self.run_targets[target].remove(fn)
459 if not self.run_targets[target]:
460 self.remove_runtarget(target, missing_list)
461
462 def remove_buildtarget(self, target, missing_list=None):
463 """
464 Mark a build target as failed (unbuildable)
465 Trigger removal of any files that have this as a dependency
466 """
467 if not missing_list:
468 missing_list = [target]
469 else:
470 missing_list = [target] + missing_list
471 logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
472 self.failed_deps.append(target)
473 dependees = self.get_dependees(target)
474 for fn in dependees:
475 self.fail_fn(fn, missing_list)
476 for tid in self.taskentries:
477 for (idepend, idependtask) in self.taskentries[tid].idepends:
478 if idepend == target:
479 fn = tid.rsplit(":",1)[0]
480 self.fail_fn(fn, missing_list)
481
482 if self.halt and target in self.external_targets:
483 logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
484 raise bb.providers.NoProvider(target)
485
486 def remove_runtarget(self, target, missing_list=None):
487 """
488 Mark a run target as failed (unbuildable)
489 Trigger removal of any files that have this as a dependency
490 """
491 if not missing_list:
492 missing_list = [target]
493 else:
494 missing_list = [target] + missing_list
495
496 logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
497 self.failed_rdeps.append(target)
498 dependees = self.get_rdependees(target)
499 for fn in dependees:
500 self.fail_fn(fn, missing_list)
501 for tid in self.taskentries:
502 for (idepend, idependtask) in self.taskentries[tid].irdepends:
503 if idepend == target:
504 fn = tid.rsplit(":",1)[0]
505 self.fail_fn(fn, missing_list)
506
507 def add_unresolved(self, cfgData, dataCache):
508 """
509 Resolve all unresolved build and runtime targets
510 """
511 logger.info("Resolving any missing task queue dependencies")
512 while True:
513 added = 0
514 for target in self.get_unresolved_build_targets(dataCache):
515 try:
516 self.add_provider_internal(cfgData, dataCache, target)
517 added = added + 1
518 except bb.providers.NoProvider:
519 if self.halt and target in self.external_targets and not self.allowincomplete:
520 raise
521 if not self.allowincomplete:
522 self.remove_buildtarget(target)
523 for target in self.get_unresolved_run_targets(dataCache):
524 try:
525 self.add_rprovider(cfgData, dataCache, target)
526 added = added + 1
527 except (bb.providers.NoRProvider, bb.providers.MultipleRProvider):
528 self.remove_runtarget(target)
529 logger.debug("Resolved " + str(added) + " extra dependencies")
530 if added == 0:
531 break
532 # self.dump_data()
533
534 def get_providermap(self, prefix=None):
535 provmap = {}
536 for name in self.build_targets:
537 if prefix and not name.startswith(prefix):
538 continue
539 if self.have_build_target(name):
540 provider = self.get_provider(name)
541 if provider:
542 provmap[name] = provider[0]
543 return provmap
544
545 def get_mcdepends(self):
546 return self.mcdepends
547
548 def dump_data(self):
549 """
550 Dump some debug information on the internal data structures
551 """
552 logger.debug3("build_names:")
553 logger.debug3(", ".join(self.build_targets))
554
555 logger.debug3("run_names:")
556 logger.debug3(", ".join(self.run_targets))
557
558 logger.debug3("build_targets:")
559 for target in self.build_targets:
560 targets = "None"
561 if target in self.build_targets:
562 targets = self.build_targets[target]
563 logger.debug3(" %s: %s", target, targets)
564
565 logger.debug3("run_targets:")
566 for target in self.run_targets:
567 targets = "None"
568 if target in self.run_targets:
569 targets = self.run_targets[target]
570 logger.debug3(" %s: %s", target, targets)
571
572 logger.debug3("tasks:")
573 for tid in self.taskentries:
574 logger.debug3(" %s: %s %s %s",
575 tid,
576 self.taskentries[tid].idepends,
577 self.taskentries[tid].irdepends,
578 self.taskentries[tid].tdepends)
579
580 logger.debug3("dependency ids (per fn):")
581 for fn in self.depids:
582 logger.debug3(" %s: %s", fn, self.depids[fn])
583
584 logger.debug3("runtime dependency ids (per fn):")
585 for fn in self.rdepids:
586 logger.debug3(" %s: %s", fn, self.rdepids[fn])
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2026-01-10 18:34:13 +0000