diff options
| -rw-r--r-- | bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py | 81 |
1 files changed, 52 insertions, 29 deletions
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py index 4c1c6a7428..27289be5fd 100644 --- a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py +++ b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py | |||
| @@ -2,18 +2,25 @@ from django.core.management.base import NoArgsCommand, CommandError | |||
| 2 | from django.db import transaction | 2 | from django.db import transaction |
| 3 | from django.db.models import Q | 3 | from django.db.models import Q |
| 4 | 4 | ||
| 5 | from bldcontrol.bbcontroller import getBuildEnvironmentController | ||
| 6 | from bldcontrol.bbcontroller import ShellCmdException, BuildSetupException | ||
| 7 | from bldcontrol.models import BuildRequest, BuildEnvironment | ||
| 8 | from bldcontrol.models import BRError, BRVariable | ||
| 9 | |||
| 5 | from orm.models import Build, ToasterSetting, LogMessage, Target | 10 | from orm.models import Build, ToasterSetting, LogMessage, Target |
| 6 | from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException | 11 | |
| 7 | from bldcontrol.models import BuildRequest, BuildEnvironment, BRError, BRVariable | ||
| 8 | import os | 12 | import os |
| 9 | import logging | 13 | import logging |
| 10 | import time | 14 | import time |
| 15 | import sys | ||
| 16 | import traceback | ||
| 11 | 17 | ||
| 12 | logger = logging.getLogger("ToasterScheduler") | 18 | logger = logging.getLogger("toaster") |
| 13 | 19 | ||
| 14 | class Command(NoArgsCommand): | 20 | class Command(NoArgsCommand): |
| 15 | args = "" | 21 | args = "" |
| 16 | help = "Schedules and executes build requests as possible. Does not return (interrupt with Ctrl-C)" | 22 | help = "Schedules and executes build requests as possible." |
| 23 | "Does not return (interrupt with Ctrl-C)" | ||
| 17 | 24 | ||
| 18 | 25 | ||
| 19 | @transaction.atomic | 26 | @transaction.atomic |
| @@ -25,21 +32,19 @@ class Command(NoArgsCommand): | |||
| 25 | 32 | ||
| 26 | @transaction.atomic | 33 | @transaction.atomic |
| 27 | def _selectBuildRequest(self): | 34 | def _selectBuildRequest(self): |
| 28 | br = BuildRequest.objects.filter(state = BuildRequest.REQ_QUEUED).order_by('pk')[0] | 35 | br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first() |
| 29 | br.state = BuildRequest.REQ_INPROGRESS | ||
| 30 | br.save() | ||
| 31 | return br | 36 | return br |
| 32 | 37 | ||
| 33 | def schedule(self): | 38 | def schedule(self): |
| 34 | import traceback | ||
| 35 | try: | 39 | try: |
| 36 | br = None | 40 | # select the build environment and the request to build |
| 37 | try: | 41 | br = self._selectBuildRequest() |
| 38 | # select the build environment and the request to build | 42 | if br: |
| 39 | br = self._selectBuildRequest() | 43 | br.state = BuildRequest.REQ_INPROGRESS |
| 40 | except IndexError as e: | 44 | br.save() |
| 41 | #logger.debug("runbuilds: No build request") | 45 | else: |
| 42 | return | 46 | return |
| 47 | |||
| 43 | try: | 48 | try: |
| 44 | bec = self._selectBuildEnvironment() | 49 | bec = self._selectBuildEnvironment() |
| 45 | except IndexError as e: | 50 | except IndexError as e: |
| @@ -49,15 +54,17 @@ class Command(NoArgsCommand): | |||
| 49 | logger.debug("runbuilds: No build env") | 54 | logger.debug("runbuilds: No build env") |
| 50 | return | 55 | return |
| 51 | 56 | ||
| 52 | logger.debug("runbuilds: starting build %s, environment %s" % (str(br).decode('utf-8'), bec.be)) | 57 | logger.debug("runbuilds: starting build %s, environment %s" % \ |
| 58 | (str(br).decode('utf-8'), bec.be)) | ||
| 53 | 59 | ||
| 54 | # let the build request know where it is being executed | 60 | # let the build request know where it is being executed |
| 55 | br.environment = bec.be | 61 | br.environment = bec.be |
| 56 | br.save() | 62 | br.save() |
| 57 | 63 | ||
| 58 | # this triggers an async build | 64 | # this triggers an async build |
| 59 | bec.triggerBuild(br.brbitbake, br.brlayer_set.all(), br.brvariable_set.all(), | 65 | bec.triggerBuild(br.brbitbake, br.brlayer_set.all(), |
| 60 | br.brtarget_set.all(), "%d:%d" % (br.pk, bec.be.pk)) | 66 | br.brvariable_set.all(), br.brtarget_set.all(), |
| 67 | "%d:%d" % (br.pk, bec.be.pk)) | ||
| 61 | 68 | ||
| 62 | except Exception as e: | 69 | except Exception as e: |
| 63 | logger.error("runbuilds: Error launching build %s" % e) | 70 | logger.error("runbuilds: Error launching build %s" % e) |
| @@ -88,9 +95,6 @@ class Command(NoArgsCommand): | |||
| 88 | def cleanup(self): | 95 | def cleanup(self): |
| 89 | from django.utils import timezone | 96 | from django.utils import timezone |
| 90 | from datetime import timedelta | 97 | from datetime import timedelta |
| 91 | # update all Builds that failed to start | ||
| 92 | |||
| 93 | for br in BuildRequest.objects.filter(state = BuildRequest.REQ_FAILED, build__outcome = Build.IN_PROGRESS): | ||
| 94 | # environments locked for more than 30 seconds | 98 | # environments locked for more than 30 seconds |
| 95 | # they should be unlocked | 99 | # they should be unlocked |
| 96 | BuildEnvironment.objects.filter( | 100 | BuildEnvironment.objects.filter( |
| @@ -102,22 +106,30 @@ class Command(NoArgsCommand): | |||
| 102 | ).update(lock=BuildEnvironment.LOCK_FREE) | 106 | ).update(lock=BuildEnvironment.LOCK_FREE) |
| 103 | 107 | ||
| 104 | 108 | ||
| 109 | # update all Builds that were in progress and failed to start | ||
| 110 | for br in BuildRequest.objects.filter( | ||
| 111 | state=BuildRequest.REQ_FAILED, | ||
| 112 | build__outcome=Build.IN_PROGRESS): | ||
| 105 | # transpose the launch errors in ToasterExceptions | 113 | # transpose the launch errors in ToasterExceptions |
| 106 | br.build.outcome = Build.FAILED | 114 | br.build.outcome = Build.FAILED |
| 107 | for brerror in br.brerror_set.all(): | 115 | for brerror in br.brerror_set.all(): |
| 108 | logger.debug("Saving error %s" % brerror) | 116 | logger.debug("Saving error %s" % brerror) |
| 109 | LogMessage.objects.create(build = br.build, level = LogMessage.EXCEPTION, message = brerror.errmsg) | 117 | LogMessage.objects.create(build=br.build, |
| 118 | level=LogMessage.EXCEPTION, | ||
| 119 | message=brerror.errmsg) | ||
| 110 | br.build.save() | 120 | br.build.save() |
| 111 | 121 | ||
| 112 | # we don't have a true build object here; hence, toasterui didn't have a change to release the BE lock | 122 | # we don't have a true build object here; hence, toasterui |
| 123 | # didn't have a change to release the BE lock | ||
| 113 | br.environment.lock = BuildEnvironment.LOCK_FREE | 124 | br.environment.lock = BuildEnvironment.LOCK_FREE |
| 114 | br.environment.save() | 125 | br.environment.save() |
| 115 | 126 | ||
| 116 | 127 | ||
| 117 | |||
| 118 | # update all BuildRequests without a build created | 128 | # update all BuildRequests without a build created |
| 119 | for br in BuildRequest.objects.filter(build = None): | 129 | for br in BuildRequest.objects.filter(build = None): |
| 120 | br.build = Build.objects.create(project = br.project, completed_on = br.updated, started_on = br.created) | 130 | br.build = Build.objects.create(project=br.project, |
| 131 | completed_on=br.updated, | ||
| 132 | started_on=br.created) | ||
| 121 | br.build.outcome = Build.FAILED | 133 | br.build.outcome = Build.FAILED |
| 122 | try: | 134 | try: |
| 123 | br.build.machine = br.brvariable_set.get(name='MACHINE').value | 135 | br.build.machine = br.brvariable_set.get(name='MACHINE').value |
| @@ -126,13 +138,16 @@ class Command(NoArgsCommand): | |||
| 126 | br.save() | 138 | br.save() |
| 127 | # transpose target information | 139 | # transpose target information |
| 128 | for brtarget in br.brtarget_set.all(): | 140 | for brtarget in br.brtarget_set.all(): |
| 129 | Target.objects.create(build=br.build, target=brtarget.target, task=brtarget.task) | 141 | Target.objects.create(build=br.build, |
| 142 | target=brtarget.target, | ||
| 143 | task=brtarget.task) | ||
| 130 | # transpose the launch errors in ToasterExceptions | 144 | # transpose the launch errors in ToasterExceptions |
| 131 | for brerror in br.brerror_set.all(): | 145 | for brerror in br.brerror_set.all(): |
| 132 | LogMessage.objects.create(build = br.build, level = LogMessage.EXCEPTION, message = brerror.errmsg) | 146 | LogMessage.objects.create(build=br.build, |
| 147 | level=LogMessage.EXCEPTION, | ||
| 148 | message=brerror.errmsg) | ||
| 133 | 149 | ||
| 134 | br.build.save() | 150 | br.build.save() |
| 135 | pass | ||
| 136 | 151 | ||
| 137 | # Make sure the LOCK is removed for builds which have been fully | 152 | # Make sure the LOCK is removed for builds which have been fully |
| 138 | # cancelled | 153 | # cancelled |
| @@ -148,9 +163,17 @@ class Command(NoArgsCommand): | |||
| 148 | while True: | 163 | while True: |
| 149 | try: | 164 | try: |
| 150 | self.cleanup() | 165 | self.cleanup() |
| 166 | except Exception as e: | ||
| 167 | logger.warn("runbuilds: cleanup exception %s" % str(e)) | ||
| 168 | |||
| 169 | try: | ||
| 151 | self.archive() | 170 | self.archive() |
| 171 | except Exception as e: | ||
| 172 | logger.warn("runbuilds: archive exception %s" % str(e)) | ||
| 173 | |||
| 174 | try: | ||
| 152 | self.schedule() | 175 | self.schedule() |
| 153 | except: | 176 | except Exception as e: |
| 154 | pass | 177 | logger.warn("runbuilds: schedule exception %s" % str(e)) |
| 155 | 178 | ||
| 156 | time.sleep(1) | 179 | time.sleep(1) |
