summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorAlexandru DAMIAN <alexandru.damian@intel.com>2013-11-26 18:12:43 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2013-12-10 11:16:12 +0000
commit54d0e30433c249604611367cf387bc20721c4523 (patch)
tree8f576220f0a4627a763d663089c9ebc0f4b29884 /bitbake/lib/bb
parentf8120984f459d193ce5ffa243137baf0e38d223e (diff)
downloadpoky-54d0e30433c249604611367cf387bc20721c4523.tar.gz
bitbake: toaster: change package storage model
Up until this patch, package information lived in two places - one table for build packages and one table for target installed packaged. This situation leads to two problems: there is no direct link between a build package and a installed package, and a lot of data is duplicated. This change unifies all package types in a single table. The SimpleUI remains the same for continuity sake, but the REST API will be changed in a future patch. The package dependencies and package files are now kept in a single table. Since we collect target installed package information at all times, we need to expand it to supplement missing information if a package is not actually built in the current build. Small changes to the Simple UI reflect the updated database schema. [YOCTO #5565] [YOCTO #5269] (Bitbake rev: f5d655bfaeb349c8680d74530617e34aa389d1f0) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/ui/buildinfohelper.py165
-rw-r--r--bitbake/lib/bb/ui/toasterui.py3
2 files changed, 78 insertions, 90 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index fef849d9a0..0252efdfef 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -20,15 +20,16 @@ import datetime
20import sys 20import sys
21import bb 21import bb
22import re 22import re
23import ast
23 24
24os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toaster.toastermain.settings") 25os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toaster.toastermain.settings")
25 26
26import toaster.toastermain.settings as toaster_django_settings 27import toaster.toastermain.settings as toaster_django_settings
27from toaster.orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage 28from toaster.orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage
28from toaster.orm.models import Variable, VariableHistory 29from toaster.orm.models import Variable, VariableHistory
29from toaster.orm.models import Target_Package, Build_Package, Build_File 30from toaster.orm.models import Package, Package_File, Target_Installed_Package
30from toaster.orm.models import Task_Dependency, Build_Package_Dependency 31from toaster.orm.models import Task_Dependency, Package_Dependency
31from toaster.orm.models import Target_Package_Dependency, Recipe_Dependency 32from toaster.orm.models import Recipe_Dependency
32from bb.msg import BBLogFormatter as format 33from bb.msg import BBLogFormatter as format
33 34
34class ORMWrapper(object): 35class ORMWrapper(object):
@@ -148,21 +149,48 @@ class ORMWrapper(object):
148 return layer_object[0] 149 return layer_object[0]
149 150
150 151
151 def save_target_package_information(self, target_obj, packagedict, bldpkgs, recipes): 152 def save_target_package_information(self, build_obj, target_obj, packagedict, pkgpnmap, recipes):
152 for p in packagedict: 153 for p in packagedict:
153 packagedict[p]['object'] = Target_Package.objects.create( target = target_obj, 154 packagedict[p]['object'], created = Package.objects.get_or_create( build = build_obj, name = p )
154 name = p, 155 if created:
155 size = packagedict[p]['size']) 156 # package was not build in the current build, but
156 if p in bldpkgs: 157 # fill in everything we can from the runtime-reverse package data
157 packagedict[p]['object'].version = bldpkgs[p]['version'] 158 try:
158 packagedict[p]['object'].recipe = recipes[bldpkgs[p]['pn']] 159 packagedict[p]['object'].recipe = recipes[pkgpnmap[p]['PN']]
159 packagedict[p]['object'].save() 160 packagedict[p]['object'].version = pkgpnmap[p]['PV']
161 packagedict[p]['object'].revision = pkgpnmap[p]['PR']
162 packagedict[p]['object'].license = pkgpnmap[p]['LICENSE']
163 packagedict[p]['object'].section = pkgpnmap[p]['SECTION']
164 packagedict[p]['object'].summary = pkgpnmap[p]['SUMMARY']
165 packagedict[p]['object'].description = pkgpnmap[p]['DESCRIPTION']
166 packagedict[p]['object'].size = int(pkgpnmap[p]['PKGSIZE'])
167
168 # no files recorded for this package, so save files info
169 for targetpath in pkgpnmap[p]['FILES_INFO']:
170 targetfilesize = pkgpnmap[p]['FILES_INFO'][targetpath]
171 Package_File.objects.create( package = packagedict[p]['object'],
172 path = targetpath,
173 size = targetfilesize)
174 except KeyError as e:
175 print "Key error, package", p, "key", e
176
177 # save disk installed size
178 packagedict[p]['object'].installed_size = packagedict[p]['size']
179 packagedict[p]['object'].save()
180
181 Target_Installed_Package.objects.create(target = target_obj, package = packagedict[p]['object'])
160 182
161 for p in packagedict: 183 for p in packagedict:
162 for (px,deptype) in packagedict[p]['depends']: 184 for (px,deptype) in packagedict[p]['depends']:
163 Target_Package_Dependency.objects.create( package = packagedict[p]['object'], 185 if deptype == 'depends':
186 tdeptype = Package_Dependency.TYPE_TRDEPENDS
187 elif deptype == 'recommends':
188 tdeptype = Package_Dependency.TYPE_TRECOMMENDS
189
190 Package_Dependency.objects.create( package = packagedict[p]['object'],
164 depends_on = packagedict[px]['object'], 191 depends_on = packagedict[px]['object'],
165 dep_type = deptype); 192 dep_type = tdeptype,
193 target = target_obj);
166 194
167 195
168 def create_logmessage(self, log_information): 196 def create_logmessage(self, log_information):
@@ -180,48 +208,53 @@ class ORMWrapper(object):
180 208
181 def save_build_package_information(self, build_obj, package_info, recipes): 209 def save_build_package_information(self, build_obj, package_info, recipes):
182 # create and save the object 210 # create and save the object
183 bp_object = Build_Package.objects.create( build = build_obj, 211 bp_object, created = Package.objects.get_or_create( build = build_obj,
184 recipe = recipes[package_info['PN']], 212 name = package_info['PKG'] )
185 name = package_info['PKG'], 213
186 version = package_info['PKGV'], 214 bp_object.recipe = recipes[package_info['PN']]
187 revision = package_info['PKGR'], 215 bp_object.version = package_info['PKGV']
188 summary = package_info['SUMMARY'], 216 bp_object.revision = package_info['PKGR']
189 description = package_info['DESCRIPTION'], 217 bp_object.summary = package_info['SUMMARY']
190 size = int(package_info['PKGSIZE']), 218 bp_object.description = package_info['DESCRIPTION']
191 section = package_info['SECTION'], 219 bp_object.size = int(package_info['PKGSIZE'])
192 license = package_info['LICENSE'], 220 bp_object.section = package_info['SECTION']
193 ) 221 bp_object.license = package_info['LICENSE']
222 bp_object.save()
223
194 # save any attached file information 224 # save any attached file information
195 for path in package_info['FILES_INFO']: 225 for path in package_info['FILES_INFO']:
196 fo = Build_File.objects.create( bpackage = bp_object, 226 fo = Package_File.objects.create( package = bp_object,
197 path = path, 227 path = path,
198 size = package_info['FILES_INFO'][path] ) 228 size = package_info['FILES_INFO'][path] )
199 229
230 def _po_byname(p):
231 return Package.objects.get_or_create(build = build_obj, name = p)[0]
232
200 # save soft dependency information 233 # save soft dependency information
201 if 'RDEPENDS' in package_info and package_info['RDEPENDS']: 234 if 'RDEPENDS' in package_info and package_info['RDEPENDS']:
202 for p in bb.utils.explode_deps(package_info['RDEPENDS']): 235 for p in bb.utils.explode_deps(package_info['RDEPENDS']):
203 Build_Package_Dependency.objects.get_or_create( package = bp_object, 236 Package_Dependency.objects.get_or_create( package = bp_object,
204 depends_on = p, dep_type = Build_Package_Dependency.TYPE_RDEPENDS) 237 depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RDEPENDS)
205 if 'RPROVIDES' in package_info and package_info['RPROVIDES']: 238 if 'RPROVIDES' in package_info and package_info['RPROVIDES']:
206 for p in bb.utils.explode_deps(package_info['RPROVIDES']): 239 for p in bb.utils.explode_deps(package_info['RPROVIDES']):
207 Build_Package_Dependency.objects.get_or_create( package = bp_object, 240 Package_Dependency.objects.get_or_create( package = bp_object,
208 depends_on = p, dep_type = Build_Package_Dependency.TYPE_RPROVIDES) 241 depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RPROVIDES)
209 if 'RRECOMMENDS' in package_info and package_info['RRECOMMENDS']: 242 if 'RRECOMMENDS' in package_info and package_info['RRECOMMENDS']:
210 for p in bb.utils.explode_deps(package_info['RRECOMMENDS']): 243 for p in bb.utils.explode_deps(package_info['RRECOMMENDS']):
211 Build_Package_Dependency.objects.get_or_create( package = bp_object, 244 Package_Dependency.objects.get_or_create( package = bp_object,
212 depends_on = p, dep_type = Build_Package_Dependency.TYPE_RRECOMMENDS) 245 depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RRECOMMENDS)
213 if 'RSUGGESTS' in package_info and package_info['RSUGGESTS']: 246 if 'RSUGGESTS' in package_info and package_info['RSUGGESTS']:
214 for p in bb.utils.explode_deps(package_info['RSUGGESTS']): 247 for p in bb.utils.explode_deps(package_info['RSUGGESTS']):
215 Build_Package_Dependency.objects.get_or_create( package = bp_object, 248 Package_Dependency.objects.get_or_create( package = bp_object,
216 depends_on = p, dep_type = Build_Package_Dependency.TYPE_RSUGGESTS) 249 depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RSUGGESTS)
217 if 'RREPLACES' in package_info and package_info['RREPLACES']: 250 if 'RREPLACES' in package_info and package_info['RREPLACES']:
218 for p in bb.utils.explode_deps(package_info['RREPLACES']): 251 for p in bb.utils.explode_deps(package_info['RREPLACES']):
219 Build_Package_Dependency.objects.get_or_create( package = bp_object, 252 Package_Dependency.objects.get_or_create( package = bp_object,
220 depends_on = p, dep_type = Build_Package_Dependency.TYPE_RREPLACES) 253 depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RREPLACES)
221 if 'RCONFLICTS' in package_info and package_info['RCONFLICTS']: 254 if 'RCONFLICTS' in package_info and package_info['RCONFLICTS']:
222 for p in bb.utils.explode_deps(package_info['RCONFLICTS']): 255 for p in bb.utils.explode_deps(package_info['RCONFLICTS']):
223 Build_Package_Dependency.objects.get_or_create( package = bp_object, 256 Package_Dependency.objects.get_or_create( package = bp_object,
224 depends_on = p, dep_type = Build_Package_Dependency.TYPE_RCONFLICTS) 257 depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RCONFLICTS)
225 258
226 return bp_object 259 return bp_object
227 260
@@ -469,54 +502,13 @@ class BuildInfoHelper(object):
469 self.orm_wrapper.get_update_task_object(task_information) 502 self.orm_wrapper.get_update_task_object(task_information)
470 503
471 504
472 def read_target_package_dep_data(self, event): 505 def store_target_package_data(self, event):
473 # for all targets 506 # for all image targets
474 for target in self.internal_state['targets']: 507 for target in self.internal_state['targets']:
475 # verify that we have something to read 508 if target.is_image:
476 if not target.is_image or not self.has_build_history: 509 pkgdata = event.data['pkgdata']
477 print "not collecting package info ", target.is_image, self.has_build_history 510 imgdata = event.data['imgdata'][target.target]
478 break 511 self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata, pkgdata, self.internal_state['recipes'])
479
480 # TODO this is a temporary replication of the code in buildhistory.bbclass
481 # This MUST be changed to query the actual BUILD_DIR_IMAGE in the target context when
482 # the capability will be implemented in Bitbake
483
484 MACHINE_ARCH, error = self.server.runCommand(['getVariable', 'MACHINE_ARCH'])
485 TCLIBC, error = self.server.runCommand(['getVariable', 'TCLIBC'])
486 BUILDHISTORY_DIR, error = self.server.runCommand(['getVariable', 'BUILDHISTORY_DIR'])
487 BUILDHISTORY_DIR_IMAGE = "%s/images/%s/%s/%s" % (BUILDHISTORY_DIR, MACHINE_ARCH, TCLIBC, target.target)
488
489 self.internal_state['packages'] = {}
490
491 with open("%s/installed-package-sizes.txt" % BUILDHISTORY_DIR_IMAGE, "r") as fin:
492 for line in fin:
493 line = line.rstrip(";")
494 psize, px = line.split("\t")
495 punit, pname = px.split(" ")
496 self.internal_state['packages'][pname.strip()] = {'size':int(psize)*1024, 'depends' : []}
497
498 with open("%s/depends.dot" % BUILDHISTORY_DIR_IMAGE, "r") as fin:
499 p = re.compile(r' -> ')
500 dot = re.compile(r'.*style=dotted')
501 for line in fin:
502 line = line.rstrip(';')
503 linesplit = p.split(line)
504 if len(linesplit) == 2:
505 pname = linesplit[0].rstrip('"').strip('"')
506 dependsname = linesplit[1].split(" ")[0].strip().strip(";").strip('"').rstrip('"')
507 deptype = Target_Package_Dependency.TYPE_DEPENDS
508 if dot.match(line):
509 deptype = Target_Package_Dependency.TYPE_RECOMMENDS
510 if not pname in self.internal_state['packages']:
511 self.internal_state['packages'][pname] = {'size': 0, 'depends' : []}
512 if not dependsname in self.internal_state['packages']:
513 self.internal_state['packages'][dependsname] = {'size': 0, 'depends' : []}
514 self.internal_state['packages'][pname]['depends'].append((dependsname, deptype))
515
516 self.orm_wrapper.save_target_package_information(target,
517 self.internal_state['packages'],
518 self.internal_state['bldpkgs'], self.internal_state['recipes'])
519
520 512
521 def store_dependency_information(self, event): 513 def store_dependency_information(self, event):
522 # save layer version priorities 514 # save layer version priorities
@@ -528,11 +520,6 @@ class BuildInfoHelper(object):
528 layer_version_obj.priority = priority 520 layer_version_obj.priority = priority
529 layer_version_obj.save() 521 layer_version_obj.save()
530 522
531 # save build time package information
532 self.internal_state['bldpkgs'] = {}
533 for pkg in event._depgraph['packages']:
534 self.internal_state['bldpkgs'][pkg] = event._depgraph['packages'][pkg]
535
536 # save recipe information 523 # save recipe information
537 self.internal_state['recipes'] = {} 524 self.internal_state['recipes'] = {}
538 for pn in event._depgraph['pn']: 525 for pn in event._depgraph['pn']:
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index 318fc28adb..e469d93e82 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -209,7 +209,6 @@ def main(server, eventHandler, params ):
209 continue 209 continue
210 210
211 if isinstance(event, (bb.event.BuildCompleted)): 211 if isinstance(event, (bb.event.BuildCompleted)):
212 buildinfohelper.read_target_package_dep_data(event)
213 buildinfohelper.update_build_information(event, errors, warnings, taskfailures) 212 buildinfohelper.update_build_information(event, errors, warnings, taskfailures)
214 continue 213 continue
215 214
@@ -240,6 +239,8 @@ def main(server, eventHandler, params ):
240 buildinfohelper.store_layer_info(event) 239 buildinfohelper.store_layer_info(event)
241 if event.type == "BuildStatsList": 240 if event.type == "BuildStatsList":
242 buildinfohelper.store_tasks_stats(event) 241 buildinfohelper.store_tasks_stats(event)
242 if event.type == "ImagePkgList":
243 buildinfohelper.store_target_package_data(event)
243 continue 244 continue
244 245
245 # ignore 246 # ignore