summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorElliot Smith <elliot.smith@intel.com>2016-07-12 15:54:44 -0700
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-07-19 08:56:51 +0100
commit5dfa120a7cf3c2cf46f165df25854e45a3b4e2f6 (patch)
tree4f6793606893d29f810d6f567d88089ec5cb4b6e /bitbake
parentb0585e6b0c8d4b9a7d2102e615fff0ef069e0f98 (diff)
downloadpoky-5dfa120a7cf3c2cf46f165df25854e45a3b4e2f6.tar.gz
bitbake: toaster: do image and artifact scan on BuildCompleted
Move the image and artifact scan code from toaster.bbclass and consolidate its logic with the existing logic in buildinfohelper. Remove handler setup for events which used to be fired from toaster.bbclass but which are now handled directly by buildinfohelper. [YOCTO #8556] (Bitbake rev: f0085cd554604cfff4a3f40a34825fbb6878004f) Signed-off-by: Elliot Smith <elliot.smith@intel.com> Signed-off-by: bavery <brian.avery@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/ui/buildinfohelper.py201
-rw-r--r--bitbake/lib/bb/ui/toasterui.py6
2 files changed, 181 insertions, 26 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 447670cb8b..8bdc9cc0a7 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -678,6 +678,16 @@ class ORMWrapper(object):
678 file_name = file_name, 678 file_name = file_name,
679 file_size = file_size) 679 file_size = file_size)
680 680
681 def save_artifact_information_no_dedupe(self, build_obj, file_name, file_size):
682 """
683 Save artifact information without checking for duplicate paths;
684 this is used when we are saving data about an artifact which was
685 generated by a previous build but which is also relevant to this build,
686 e.g. a bzImage file.
687 """
688 BuildArtifact.objects.create(build=build_obj, file_name=file_name,
689 file_size=file_size)
690
681 def save_artifact_information(self, build_obj, file_name, file_size): 691 def save_artifact_information(self, build_obj, file_name, file_size):
682 # we skip the image files from other builds 692 # we skip the image files from other builds
683 if Target_Image_File.objects.filter(file_name = file_name).count() > 0: 693 if Target_Image_File.objects.filter(file_name = file_name).count() > 0:
@@ -687,7 +697,8 @@ class ORMWrapper(object):
687 if BuildArtifact.objects.filter(file_name = file_name).count() > 0: 697 if BuildArtifact.objects.filter(file_name = file_name).count() > 0:
688 return 698 return
689 699
690 BuildArtifact.objects.create(build = build_obj, file_name = file_name, file_size = file_size) 700 self.save_artifact_information_no_dedupe(self, build_obj, file_name,
701 file_size)
691 702
692 def create_logmessage(self, log_information): 703 def create_logmessage(self, log_information):
693 assert 'build' in log_information 704 assert 'build' in log_information
@@ -1061,17 +1072,6 @@ class BuildInfoHelper(object):
1061 1072
1062 return self.brbe 1073 return self.brbe
1063 1074
1064
1065 def update_target_image_file(self, event):
1066 evdata = BuildInfoHelper._get_data_from_event(event)
1067
1068 for t in self.internal_state['targets']:
1069 if t.is_image == True:
1070 output_files = list(evdata.keys())
1071 for output in output_files:
1072 if t.target in output and 'rootfs' in output and not output.endswith(".manifest"):
1073 self.orm_wrapper.save_target_image_file_information(t, output, evdata[output])
1074
1075 def update_artifact_image_file(self, event): 1075 def update_artifact_image_file(self, event):
1076 evdata = BuildInfoHelper._get_data_from_event(event) 1076 evdata = BuildInfoHelper._get_data_from_event(event)
1077 for artifact_path in evdata.keys(): 1077 for artifact_path in evdata.keys():
@@ -1081,16 +1081,6 @@ class BuildInfoHelper(object):
1081 if 'build' in self.internal_state: 1081 if 'build' in self.internal_state:
1082 self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) 1082 self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)
1083 1083
1084
1085 def store_license_manifest_path(self, event):
1086 deploy_dir = BuildInfoHelper._get_data_from_event(event)['deploy_dir']
1087 image_name = BuildInfoHelper._get_data_from_event(event)['image_name']
1088 path = deploy_dir + "/licenses/" + image_name + "/license.manifest"
1089 for target in self.internal_state['targets']:
1090 if target.target in image_name:
1091 self.orm_wrapper.update_target_set_license_manifest(target, path)
1092
1093
1094 def store_started_task(self, event): 1084 def store_started_task(self, event):
1095 assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped)) 1085 assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped))
1096 assert 'taskfile' in vars(event) 1086 assert 'taskfile' in vars(event)
@@ -1506,6 +1496,173 @@ class BuildInfoHelper(object):
1506 1496
1507 self.orm_wrapper.create_logmessage(log_information) 1497 self.orm_wrapper.create_logmessage(log_information)
1508 1498
1499 def _get_files_from_image_license(self, image_license_manifest_path):
1500 """
1501 Find the FILES line in the image_license.manifest file,
1502 which has the basenames of the bzImage and modules files
1503 in this format:
1504 FILES: bzImage--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.bin modules--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.tgz
1505 """
1506 files = []
1507 with open(image_license_manifest_path) as image_license:
1508 for line in image_license:
1509 if line.startswith('FILES'):
1510 files_str = line.split(':')[1].strip()
1511 files_str = re.sub(r' {2,}', ' ', files_str)
1512 files = files_str.split(' ')
1513 return files
1514
1515 def _endswith(self, str_to_test, endings):
1516 """
1517 Returns True if str ends with one of the strings in the list
1518 endings, False otherwise
1519 """
1520 endswith = False
1521 for ending in endings:
1522 if str_to_test.endswith(ending):
1523 endswith = True
1524 break
1525 return endswith
1526
1527 def _get_image_files(self, deploy_dir_image, image_name, image_file_extensions):
1528 """
1529 Find files in deploy_dir_image whose basename starts with the
1530 string image_name and ends with one of the strings in
1531 image_file_extensions.
1532
1533 Returns a list of file dictionaries like
1534
1535 [
1536 {
1537 'path': '/path/to/image/file',
1538 'size': <file size in bytes>
1539 }
1540 ]
1541 """
1542 image_files = []
1543
1544 for dirpath, _, filenames in os.walk(deploy_dir_image):
1545 for filename in filenames:
1546 if filename.startswith(image_name) and \
1547 self._endswith(filename, image_file_extensions):
1548 image_file_path = os.path.join(dirpath, filename)
1549 image_file_size = os.stat(image_file_path).st_size
1550
1551 image_files.append({
1552 'path': image_file_path,
1553 'size': image_file_size
1554 })
1555
1556 return image_files
1557
1558 def scan_build_artifacts(self):
1559 """
1560 Scan for build artifacts in DEPLOY_DIR_IMAGE and associate them
1561 with a Target object in self.internal_state['targets'].
1562
1563 We have two situations to handle:
1564
1565 1. This is the first time a target + machine has been built, so
1566 add files from the DEPLOY_DIR_IMAGE to the target.
1567
1568 OR
1569
1570 2. There are no files for the target, so copy them from a
1571 previous build with the same target + machine.
1572 """
1573 deploy_dir_image = \
1574 self.server.runCommand(['getVariable', 'DEPLOY_DIR_IMAGE'])[0]
1575
1576 # if there's no DEPLOY_DIR_IMAGE, there aren't going to be
1577 # any build artifacts, so we can return immediately
1578 if not deploy_dir_image:
1579 return
1580
1581 buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0]
1582 machine = self.server.runCommand(['getVariable', 'MACHINE'])[0]
1583 image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
1584
1585 # location of the image_license.manifest files for this build;
1586 # note that this file is only produced if an image is produced
1587 license_directory = \
1588 self.server.runCommand(['getVariable', 'LICENSE_DIRECTORY'])[0]
1589
1590 # file name extensions for image files
1591 image_file_extensions_unique = {}
1592 image_fstypes = self.server.runCommand(
1593 ['getVariable', 'IMAGE_FSTYPES'])[0]
1594 if image_fstypes != None:
1595 image_types_str = image_fstypes.strip()
1596 image_file_extensions = re.sub(r' {2,}', ' ', image_types_str)
1597 image_file_extensions_unique = set(image_file_extensions.split(' '))
1598
1599 targets = self.internal_state['targets']
1600 image_targets = [target for target in targets if target.is_image]
1601 for target in image_targets:
1602 # this is set to True if we find at least one file relating to
1603 # this target; if this remains False after the scan, we copy the
1604 # files from the most-recent Target with the same target + machine
1605 # onto this Target instead
1606 has_files = False
1607
1608 # we construct this because by the time we reach
1609 # BuildCompleted, this has reset to
1610 # 'defaultpkgname-<MACHINE>-<BUILDNAME>';
1611 # we need to change it to
1612 # <TARGET>-<MACHINE>-<BUILDNAME>
1613 real_image_name = re.sub(r'^defaultpkgname', target.target,
1614 image_name)
1615
1616 image_license_manifest_path = os.path.join(
1617 license_directory,
1618 real_image_name,
1619 'image_license.manifest')
1620
1621 # if image_license.manifest exists, we can read the names of bzImage
1622 # and modules files for this build from it, then look for them
1623 # in the DEPLOY_DIR_IMAGE; note that this file is only produced
1624 # if an image file was produced
1625 if os.path.isfile(image_license_manifest_path):
1626 has_files = True
1627
1628 basenames = self._get_files_from_image_license(
1629 image_license_manifest_path)
1630
1631 for basename in basenames:
1632 artifact_path = os.path.join(deploy_dir_image, basename)
1633 artifact_size = os.stat(artifact_path).st_size
1634
1635 self.orm_wrapper.save_artifact_information_no_dedupe(
1636 self.internal_state['build'], artifact_path,
1637 artifact_size)
1638
1639 # store the license manifest path on the target
1640 # (this file is also created any time an image file is created)
1641 license_path = os.path.join(license_directory,
1642 real_image_name, 'license.manifest')
1643
1644 self.orm_wrapper.update_target_set_license_manifest(target,
1645 license_path)
1646
1647 # scan the directory for image files relating to this build
1648 # (via real_image_name); note that we don't have to set
1649 # has_files = True, as searching for the license manifest file
1650 # will already have set it to true if at least one image file was
1651 # produced
1652 image_files = self._get_image_files(deploy_dir_image,
1653 real_image_name, image_file_extensions_unique)
1654
1655 for image_file in image_files:
1656 self.orm_wrapper.save_target_image_file_information(
1657 target, image_file['path'], image_file['size'])
1658
1659 if not has_files:
1660 # TODO copy artifact and image files from the
1661 # most-recently-built Target with the same target + machine
1662 # as this Target; also copy the license manifest path,
1663 # as that is treated differently
1664 pass
1665
1509 def close(self, errorcode): 1666 def close(self, errorcode):
1510 if self.brbe is not None: 1667 if self.brbe is not None:
1511 self._store_build_done(errorcode) 1668 self._store_build_done(errorcode)
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index 5382935f82..d8bccdb81c 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -363,6 +363,8 @@ def main(server, eventHandler, params):
363 errors += 1 363 errors += 1
364 errorcode = 1 364 errorcode = 1
365 logger.error("Command execution failed: %s", event.error) 365 logger.error("Command execution failed: %s", event.error)
366 elif isinstance(event, bb.event.BuildCompleted):
367 buildinfohelper.scan_build_artifacts()
366 368
367 # turn off logging to the current build log 369 # turn off logging to the current build log
368 _close_build_log(build_log) 370 _close_build_log(build_log)
@@ -410,12 +412,8 @@ def main(server, eventHandler, params):
410 buildinfohelper.store_target_package_data(event) 412 buildinfohelper.store_target_package_data(event)
411 elif event.type == "MissedSstate": 413 elif event.type == "MissedSstate":
412 buildinfohelper.store_missed_state_tasks(event) 414 buildinfohelper.store_missed_state_tasks(event)
413 elif event.type == "ImageFileSize":
414 buildinfohelper.update_target_image_file(event)
415 elif event.type == "ArtifactFileSize": 415 elif event.type == "ArtifactFileSize":
416 buildinfohelper.update_artifact_image_file(event) 416 buildinfohelper.update_artifact_image_file(event)
417 elif event.type == "LicenseManifestPath":
418 buildinfohelper.store_license_manifest_path(event)
419 elif event.type == "SetBRBE": 417 elif event.type == "SetBRBE":
420 buildinfohelper.brbe = buildinfohelper._get_data_from_event(event) 418 buildinfohelper.brbe = buildinfohelper._get_data_from_event(event)
421 elif event.type == "OSErrorException": 419 elif event.type == "OSErrorException":