4 files changed, 133 insertions, 6 deletions

diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
index 75674ccbf1..577e765f11 100644
--- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -200,7 +200,7 @@ class LocalhostBEController(BuildEnvironmentController):
                 localdirpath = os.path.join(localdirname, dirpath)
                 logger.debug("localhostbecontroller: localdirpath expects '%s'" % localdirpath)
                 if not os.path.exists(localdirpath):
-                    raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
+                    raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Exiting." % (localdirpath, giturl, commit))
 
                 if name != "bitbake":
                     layerlist.append("%03d:%s" % (index,localdirpath.rstrip("/")))
@@ -467,7 +467,7 @@ class LocalhostBEController(BuildEnvironmentController):
             logger.debug("localhostbecontroller: waiting for bblock content to appear")
             time.sleep(1)
         else:
-            raise BuildSetupException("Cannot find bitbake server lock file '%s'. Aborting." % bblock)
+            raise BuildSetupException("Cannot find bitbake server lock file '%s'. Exiting." % bblock)
 
         with open(bblock) as fplock:
             for line in fplock:
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
index 19f659ec41..834e32b36f 100644
--- a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
+++ b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
@@ -180,6 +180,77 @@ class Command(BaseCommand):
         except Exception as e:
             logger.warning("runbuilds: schedule exception %s" % str(e))
 
+    # Test to see if a build pre-maturely died due to a bitbake crash
+    def check_dead_builds(self):
+        do_cleanup = False
+        try:
+            for br in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS):
+                # Get the build directory
+                if br.project.builddir:
+                    builddir =  br.project.builddir
+                else:
+                    builddir = '%s-toaster-%d' % (br.environment.builddir,br.project.id)
+                # Check log to see if there is a recent traceback
+                toaster_ui_log = os.path.join(builddir, 'toaster_ui.log')
+                test_file = os.path.join(builddir, '._toaster_check.txt')
+                os.system("tail -n 50 %s > %s" % (os.path.join(builddir, 'toaster_ui.log'),test_file))
+                traceback_text = ''
+                is_traceback = False
+                with open(test_file,'r') as test_file_fd:
+                    test_file_tail = test_file_fd.readlines()
+                    for line in test_file_tail:
+                        if line.startswith('Traceback (most recent call last):'):
+                            traceback_text = line
+                            is_traceback = True
+                        elif line.startswith('NOTE: ToasterUI waiting for events'):
+                            # Ignore any traceback before new build start
+                            traceback_text = ''
+                            is_traceback = False
+                        elif line.startswith('Note: Toaster traceback auto-stop'):
+                            # Ignore any traceback before this previous traceback catch
+                            traceback_text = ''
+                            is_traceback = False
+                        elif is_traceback:
+                            traceback_text += line
+                # Test the results
+                is_stop = False
+                if is_traceback:
+                    # Found a traceback
+                    errtype = 'Bitbake crash'
+                    errmsg = 'Bitbake crash\n' + traceback_text
+                    state = BuildRequest.REQ_FAILED
+                    # Clean up bitbake files
+                    bitbake_lock = os.path.join(builddir, 'bitbake.lock')
+                    if os.path.isfile(bitbake_lock):
+                        os.remove(bitbake_lock)
+                    bitbake_sock = os.path.join(builddir, 'bitbake.sock')
+                    if os.path.isfile(bitbake_sock):
+                        os.remove(bitbake_sock)
+                    if os.path.isfile(test_file):
+                        os.remove(test_file)
+                    # Add note to ignore this traceback on next check
+                    os.system('echo "Note: Toaster traceback auto-stop" >> %s' % toaster_ui_log)
+                    is_stop = True
+                # Add more tests here
+                #elif ...
+                # Stop the build request?
+                if is_stop:
+                    brerror = BRError(
+                        req = br,
+                        errtype = errtype,
+                        errmsg = errmsg,
+                        traceback = traceback_text,
+                        )
+                    brerror.save()
+                    br.state = state
+                    br.save()
+                    do_cleanup = True
+            # Do cleanup
+            if do_cleanup:
+                self.cleanup()
+        except Exception as e:
+            logger.error("runbuilds: Error in check_dead_builds %s" % e)
+
     def handle(self, **options):
         pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
                                     ".runbuilds.pid")
@@ -187,10 +258,18 @@ class Command(BaseCommand):
         with open(pidfile_path, 'w') as pidfile:
             pidfile.write("%s" % os.getpid())
 
+        # Clean up any stale/failed builds from previous Toaster run
         self.runbuild()
 
         signal.signal(signal.SIGUSR1, lambda sig, frame: None)
 
         while True:
-            signal.pause()
-            self.runbuild()
+            sigset = signal.sigtimedwait([signal.SIGUSR1], 5)
+            if sigset:
+                for sig in sigset:
+                    # Consume each captured pending event
+                    self.runbuild()
+            else:
+                # Check for build exceptions
+                self.check_dead_builds()
+
diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py b/bitbake/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py
new file mode 100644
index 0000000000..45b477d02c
--- /dev/null
+++ b/bitbake/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py
@@ -0,0 +1,48 @@
+# Generated by Django 3.2.12 on 2022-03-06 03:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bldcontrol', '0007_brlayers_optional_gitinfo'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='brbitbake',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='brerror',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='brlayer',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='brtarget',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='brvariable',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='buildenvironment',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='buildrequest',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+    ]
diff --git a/bitbake/lib/toaster/bldcontrol/models.py b/bitbake/lib/toaster/bldcontrol/models.py
index c2f302da24..42750e7180 100644
--- a/bitbake/lib/toaster/bldcontrol/models.py
+++ b/bitbake/lib/toaster/bldcontrol/models.py
@@ -4,7 +4,7 @@
 
 from __future__ import unicode_literals
 from django.db import models
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 from orm.models import Project, Build, Layer_Version
 
 import logging
@@ -124,7 +124,7 @@ class BuildRequest(models.Model):
         return self.brvariable_set.get(name="MACHINE").value
 
     def __str__(self):
-        return force_text('%s %s' % (self.project, self.get_state_display()))
+        return force_str('%s %s' % (self.project, self.get_state_display()))
 
 # These tables specify the settings for running an actual build.
 # They MUST be kept in sync with the tables in orm.models.Project*