diff options
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r-- | bitbake/lib/bb/ui/buildinfohelper.py | 201 | ||||
-rw-r--r-- | bitbake/lib/bb/ui/toasterui.py | 6 |
2 files changed, 181 insertions, 26 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py index 447670cb8b..8bdc9cc0a7 100644 --- a/bitbake/lib/bb/ui/buildinfohelper.py +++ b/bitbake/lib/bb/ui/buildinfohelper.py | |||
@@ -678,6 +678,16 @@ class ORMWrapper(object): | |||
678 | file_name = file_name, | 678 | file_name = file_name, |
679 | file_size = file_size) | 679 | file_size = file_size) |
680 | 680 | ||
681 | def save_artifact_information_no_dedupe(self, build_obj, file_name, file_size): | ||
682 | """ | ||
683 | Save artifact information without checking for duplicate paths; | ||
684 | this is used when we are saving data about an artifact which was | ||
685 | generated by a previous build but which is also relevant to this build, | ||
686 | e.g. a bzImage file. | ||
687 | """ | ||
688 | BuildArtifact.objects.create(build=build_obj, file_name=file_name, | ||
689 | file_size=file_size) | ||
690 | |||
681 | def save_artifact_information(self, build_obj, file_name, file_size): | 691 | def save_artifact_information(self, build_obj, file_name, file_size): |
682 | # we skip the image files from other builds | 692 | # we skip the image files from other builds |
683 | if Target_Image_File.objects.filter(file_name = file_name).count() > 0: | 693 | if Target_Image_File.objects.filter(file_name = file_name).count() > 0: |
@@ -687,7 +697,8 @@ class ORMWrapper(object): | |||
687 | if BuildArtifact.objects.filter(file_name = file_name).count() > 0: | 697 | if BuildArtifact.objects.filter(file_name = file_name).count() > 0: |
688 | return | 698 | return |
689 | 699 | ||
690 | BuildArtifact.objects.create(build = build_obj, file_name = file_name, file_size = file_size) | 700 | self.save_artifact_information_no_dedupe(self, build_obj, file_name, |
701 | file_size) | ||
691 | 702 | ||
692 | def create_logmessage(self, log_information): | 703 | def create_logmessage(self, log_information): |
693 | assert 'build' in log_information | 704 | assert 'build' in log_information |
@@ -1061,17 +1072,6 @@ class BuildInfoHelper(object): | |||
1061 | 1072 | ||
1062 | return self.brbe | 1073 | return self.brbe |
1063 | 1074 | ||
1064 | |||
1065 | def update_target_image_file(self, event): | ||
1066 | evdata = BuildInfoHelper._get_data_from_event(event) | ||
1067 | |||
1068 | for t in self.internal_state['targets']: | ||
1069 | if t.is_image == True: | ||
1070 | output_files = list(evdata.keys()) | ||
1071 | for output in output_files: | ||
1072 | if t.target in output and 'rootfs' in output and not output.endswith(".manifest"): | ||
1073 | self.orm_wrapper.save_target_image_file_information(t, output, evdata[output]) | ||
1074 | |||
1075 | def update_artifact_image_file(self, event): | 1075 | def update_artifact_image_file(self, event): |
1076 | evdata = BuildInfoHelper._get_data_from_event(event) | 1076 | evdata = BuildInfoHelper._get_data_from_event(event) |
1077 | for artifact_path in evdata.keys(): | 1077 | for artifact_path in evdata.keys(): |
@@ -1081,16 +1081,6 @@ class BuildInfoHelper(object): | |||
1081 | if 'build' in self.internal_state: | 1081 | if 'build' in self.internal_state: |
1082 | self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) | 1082 | self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) |
1083 | 1083 | ||
1084 | |||
1085 | def store_license_manifest_path(self, event): | ||
1086 | deploy_dir = BuildInfoHelper._get_data_from_event(event)['deploy_dir'] | ||
1087 | image_name = BuildInfoHelper._get_data_from_event(event)['image_name'] | ||
1088 | path = deploy_dir + "/licenses/" + image_name + "/license.manifest" | ||
1089 | for target in self.internal_state['targets']: | ||
1090 | if target.target in image_name: | ||
1091 | self.orm_wrapper.update_target_set_license_manifest(target, path) | ||
1092 | |||
1093 | |||
1094 | def store_started_task(self, event): | 1084 | def store_started_task(self, event): |
1095 | assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped)) | 1085 | assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped)) |
1096 | assert 'taskfile' in vars(event) | 1086 | assert 'taskfile' in vars(event) |
@@ -1506,6 +1496,173 @@ class BuildInfoHelper(object): | |||
1506 | 1496 | ||
1507 | self.orm_wrapper.create_logmessage(log_information) | 1497 | self.orm_wrapper.create_logmessage(log_information) |
1508 | 1498 | ||
1499 | def _get_files_from_image_license(self, image_license_manifest_path): | ||
1500 | """ | ||
1501 | Find the FILES line in the image_license.manifest file, | ||
1502 | which has the basenames of the bzImage and modules files | ||
1503 | in this format: | ||
1504 | FILES: bzImage--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.bin modules--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.tgz | ||
1505 | """ | ||
1506 | files = [] | ||
1507 | with open(image_license_manifest_path) as image_license: | ||
1508 | for line in image_license: | ||
1509 | if line.startswith('FILES'): | ||
1510 | files_str = line.split(':')[1].strip() | ||
1511 | files_str = re.sub(r' {2,}', ' ', files_str) | ||
1512 | files = files_str.split(' ') | ||
1513 | return files | ||
1514 | |||
1515 | def _endswith(self, str_to_test, endings): | ||
1516 | """ | ||
1517 | Returns True if str ends with one of the strings in the list | ||
1518 | endings, False otherwise | ||
1519 | """ | ||
1520 | endswith = False | ||
1521 | for ending in endings: | ||
1522 | if str_to_test.endswith(ending): | ||
1523 | endswith = True | ||
1524 | break | ||
1525 | return endswith | ||
1526 | |||
1527 | def _get_image_files(self, deploy_dir_image, image_name, image_file_extensions): | ||
1528 | """ | ||
1529 | Find files in deploy_dir_image whose basename starts with the | ||
1530 | string image_name and ends with one of the strings in | ||
1531 | image_file_extensions. | ||
1532 | |||
1533 | Returns a list of file dictionaries like | ||
1534 | |||
1535 | [ | ||
1536 | { | ||
1537 | 'path': '/path/to/image/file', | ||
1538 | 'size': <file size in bytes> | ||
1539 | } | ||
1540 | ] | ||
1541 | """ | ||
1542 | image_files = [] | ||
1543 | |||
1544 | for dirpath, _, filenames in os.walk(deploy_dir_image): | ||
1545 | for filename in filenames: | ||
1546 | if filename.startswith(image_name) and \ | ||
1547 | self._endswith(filename, image_file_extensions): | ||
1548 | image_file_path = os.path.join(dirpath, filename) | ||
1549 | image_file_size = os.stat(image_file_path).st_size | ||
1550 | |||
1551 | image_files.append({ | ||
1552 | 'path': image_file_path, | ||
1553 | 'size': image_file_size | ||
1554 | }) | ||
1555 | |||
1556 | return image_files | ||
1557 | |||
1558 | def scan_build_artifacts(self): | ||
1559 | """ | ||
1560 | Scan for build artifacts in DEPLOY_DIR_IMAGE and associate them | ||
1561 | with a Target object in self.internal_state['targets']. | ||
1562 | |||
1563 | We have two situations to handle: | ||
1564 | |||
1565 | 1. This is the first time a target + machine has been built, so | ||
1566 | add files from the DEPLOY_DIR_IMAGE to the target. | ||
1567 | |||
1568 | OR | ||
1569 | |||
1570 | 2. There are no files for the target, so copy them from a | ||
1571 | previous build with the same target + machine. | ||
1572 | """ | ||
1573 | deploy_dir_image = \ | ||
1574 | self.server.runCommand(['getVariable', 'DEPLOY_DIR_IMAGE'])[0] | ||
1575 | |||
1576 | # if there's no DEPLOY_DIR_IMAGE, there aren't going to be | ||
1577 | # any build artifacts, so we can return immediately | ||
1578 | if not deploy_dir_image: | ||
1579 | return | ||
1580 | |||
1581 | buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0] | ||
1582 | machine = self.server.runCommand(['getVariable', 'MACHINE'])[0] | ||
1583 | image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0] | ||
1584 | |||
1585 | # location of the image_license.manifest files for this build; | ||
1586 | # note that this file is only produced if an image is produced | ||
1587 | license_directory = \ | ||
1588 | self.server.runCommand(['getVariable', 'LICENSE_DIRECTORY'])[0] | ||
1589 | |||
1590 | # file name extensions for image files | ||
1591 | image_file_extensions_unique = {} | ||
1592 | image_fstypes = self.server.runCommand( | ||
1593 | ['getVariable', 'IMAGE_FSTYPES'])[0] | ||
1594 | if image_fstypes != None: | ||
1595 | image_types_str = image_fstypes.strip() | ||
1596 | image_file_extensions = re.sub(r' {2,}', ' ', image_types_str) | ||
1597 | image_file_extensions_unique = set(image_file_extensions.split(' ')) | ||
1598 | |||
1599 | targets = self.internal_state['targets'] | ||
1600 | image_targets = [target for target in targets if target.is_image] | ||
1601 | for target in image_targets: | ||
1602 | # this is set to True if we find at least one file relating to | ||
1603 | # this target; if this remains False after the scan, we copy the | ||
1604 | # files from the most-recent Target with the same target + machine | ||
1605 | # onto this Target instead | ||
1606 | has_files = False | ||
1607 | |||
1608 | # we construct this because by the time we reach | ||
1609 | # BuildCompleted, this has reset to | ||
1610 | # 'defaultpkgname-<MACHINE>-<BUILDNAME>'; | ||
1611 | # we need to change it to | ||
1612 | # <TARGET>-<MACHINE>-<BUILDNAME> | ||
1613 | real_image_name = re.sub(r'^defaultpkgname', target.target, | ||
1614 | image_name) | ||
1615 | |||
1616 | image_license_manifest_path = os.path.join( | ||
1617 | license_directory, | ||
1618 | real_image_name, | ||
1619 | 'image_license.manifest') | ||
1620 | |||
1621 | # if image_license.manifest exists, we can read the names of bzImage | ||
1622 | # and modules files for this build from it, then look for them | ||
1623 | # in the DEPLOY_DIR_IMAGE; note that this file is only produced | ||
1624 | # if an image file was produced | ||
1625 | if os.path.isfile(image_license_manifest_path): | ||
1626 | has_files = True | ||
1627 | |||
1628 | basenames = self._get_files_from_image_license( | ||
1629 | image_license_manifest_path) | ||
1630 | |||
1631 | for basename in basenames: | ||
1632 | artifact_path = os.path.join(deploy_dir_image, basename) | ||
1633 | artifact_size = os.stat(artifact_path).st_size | ||
1634 | |||
1635 | self.orm_wrapper.save_artifact_information_no_dedupe( | ||
1636 | self.internal_state['build'], artifact_path, | ||
1637 | artifact_size) | ||
1638 | |||
1639 | # store the license manifest path on the target | ||
1640 | # (this file is also created any time an image file is created) | ||
1641 | license_path = os.path.join(license_directory, | ||
1642 | real_image_name, 'license.manifest') | ||
1643 | |||
1644 | self.orm_wrapper.update_target_set_license_manifest(target, | ||
1645 | license_path) | ||
1646 | |||
1647 | # scan the directory for image files relating to this build | ||
1648 | # (via real_image_name); note that we don't have to set | ||
1649 | # has_files = True, as searching for the license manifest file | ||
1650 | # will already have set it to true if at least one image file was | ||
1651 | # produced | ||
1652 | image_files = self._get_image_files(deploy_dir_image, | ||
1653 | real_image_name, image_file_extensions_unique) | ||
1654 | |||
1655 | for image_file in image_files: | ||
1656 | self.orm_wrapper.save_target_image_file_information( | ||
1657 | target, image_file['path'], image_file['size']) | ||
1658 | |||
1659 | if not has_files: | ||
1660 | # TODO copy artifact and image files from the | ||
1661 | # most-recently-built Target with the same target + machine | ||
1662 | # as this Target; also copy the license manifest path, | ||
1663 | # as that is treated differently | ||
1664 | pass | ||
1665 | |||
1509 | def close(self, errorcode): | 1666 | def close(self, errorcode): |
1510 | if self.brbe is not None: | 1667 | if self.brbe is not None: |
1511 | self._store_build_done(errorcode) | 1668 | self._store_build_done(errorcode) |
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py index 5382935f82..d8bccdb81c 100644 --- a/bitbake/lib/bb/ui/toasterui.py +++ b/bitbake/lib/bb/ui/toasterui.py | |||
@@ -363,6 +363,8 @@ def main(server, eventHandler, params): | |||
363 | errors += 1 | 363 | errors += 1 |
364 | errorcode = 1 | 364 | errorcode = 1 |
365 | logger.error("Command execution failed: %s", event.error) | 365 | logger.error("Command execution failed: %s", event.error) |
366 | elif isinstance(event, bb.event.BuildCompleted): | ||
367 | buildinfohelper.scan_build_artifacts() | ||
366 | 368 | ||
367 | # turn off logging to the current build log | 369 | # turn off logging to the current build log |
368 | _close_build_log(build_log) | 370 | _close_build_log(build_log) |
@@ -410,12 +412,8 @@ def main(server, eventHandler, params): | |||
410 | buildinfohelper.store_target_package_data(event) | 412 | buildinfohelper.store_target_package_data(event) |
411 | elif event.type == "MissedSstate": | 413 | elif event.type == "MissedSstate": |
412 | buildinfohelper.store_missed_state_tasks(event) | 414 | buildinfohelper.store_missed_state_tasks(event) |
413 | elif event.type == "ImageFileSize": | ||
414 | buildinfohelper.update_target_image_file(event) | ||
415 | elif event.type == "ArtifactFileSize": | 415 | elif event.type == "ArtifactFileSize": |
416 | buildinfohelper.update_artifact_image_file(event) | 416 | buildinfohelper.update_artifact_image_file(event) |
417 | elif event.type == "LicenseManifestPath": | ||
418 | buildinfohelper.store_license_manifest_path(event) | ||
419 | elif event.type == "SetBRBE": | 417 | elif event.type == "SetBRBE": |
420 | buildinfohelper.brbe = buildinfohelper._get_data_from_event(event) | 418 | buildinfohelper.brbe = buildinfohelper._get_data_from_event(event) |
421 | elif event.type == "OSErrorException": | 419 | elif event.type == "OSErrorException": |