patchtest: clean up test suite

Various tweaks to make the test suite cleaner and more efficient:

- Replace use of "re" module with "pyparsing" in tests (but not base.py)
- Make test_mbox_cve only check for CVE tags in the commit if the added
  patch has them
- Make test_mbox_cve SKIP instead of PASS if there's no CVE tag
- Simplify the bugzilla tag checking test now that pyparsing is used
- Modify the selftest script to correctly parse the new result output

(From OE-Core rev: 7a187c2475aa762e2bc830950f608143f2535a72)

Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

12 files changed, 45 insertions, 44 deletions

diff --git a/meta/lib/patchtest/selftest/selftest b/meta/lib/patchtest/selftest/selftest
index d2b61e951a..f8985314f5 100755
--- a/meta/lib/patchtest/selftest/selftest
+++ b/meta/lib/patchtest/selftest/selftest
@@ -63,7 +63,7 @@ if __name__ == '__main__':
 
             for resultline in results.splitlines():
                 if testid in resultline:
-                    result, _ = resultline.split(' ', 1)
+                    result, _ = resultline.split(':', 1)
 
                     if expected_result.upper() == "FAIL" and result.upper() == "FAIL":
                         xfailcount = xfailcount + 1
diff --git a/meta/lib/patchtest/tests/test_mbox_author.py b/meta/lib/patchtest/tests/test_mbox_author.py
index fb8f10e1fd..e68e7a5ac4 100644
--- a/meta/lib/patchtest/tests/test_mbox_author.py
+++ b/meta/lib/patchtest/tests/test_mbox_author.py
@@ -5,22 +5,22 @@
 # SPDX-License-Identifier: GPL-2.0
 
 import base
-import re
+import pyparsing
 
 class Author(base.Base):
 
-    auh_email = '<auh@auh.yoctoproject.org>'
+    auh_email = 'auh@auh.yoctoproject.org'
 
-    invalids = [re.compile("^Upgrade Helper.+"),
-                re.compile(re.escape(auh_email)),
-                re.compile("uh@not\.set"),
-                re.compile("\S+@example\.com")]
+    invalids = [pyparsing.Regex("^Upgrade Helper.+"),
+                pyparsing.Regex(auh_email),
+                pyparsing.Regex("uh@not\.set"),
+                pyparsing.Regex("\S+@example\.com")]
 
 
     def test_author_valid(self):
         for commit in self.commits:
             for invalid in self.invalids:
-                if invalid.search(commit.author):
+                if invalid.search_string(commit.author):
                     self.fail('Invalid author %s. Resend the series with a valid patch author' % commit.author, commit=commit)
 
     def test_non_auh_upgrade(self):
diff --git a/meta/lib/patchtest/tests/test_mbox_bugzilla.py b/meta/lib/patchtest/tests/test_mbox_bugzilla.py
index aa53b77f87..adf46b5d59 100644
--- a/meta/lib/patchtest/tests/test_mbox_bugzilla.py
+++ b/meta/lib/patchtest/tests/test_mbox_bugzilla.py
@@ -4,17 +4,17 @@
 #
 # SPDX-License-Identifier: GPL-2.0
 
-import re
+import pyparsing
 import base
 
 class Bugzilla(base.Base):
-    rexp_detect     = re.compile("\[\s?YOCTO.*\]", re.IGNORECASE)
-    rexp_validation = re.compile("\[(\s?YOCTO\s?#\s?(\d+)\s?,?)+\]", re.IGNORECASE)
+    rexp_detect = pyparsing.Regex('\[\s?YOCTO.*\]')
+    rexp_validation = pyparsing.Regex('\[(\s?YOCTO\s?#\s?(\d+)\s?,?)+\]')
 
     def test_bugzilla_entry_format(self):
         for commit in Bugzilla.commits:
-            for line in commit.commit_message.splitlines():
-                if self.rexp_detect.match(line):
-                    if not self.rexp_validation.match(line):
-                        self.fail('Bugzilla issue ID is not correctly formatted - specify it with format: "[YOCTO #<bugzilla ID>]"', commit=commit)
+            if not self.rexp_detect.search_string(commit.commit_message):
+                self.skip("No bug ID found")
+            elif not self.rexp_validation.search_string(commit.commit_message):
+                self.fail('Bugzilla issue ID is not correctly formatted - specify it with format: "[YOCTO #<bugzilla ID>]"', commit=commit)
 
diff --git a/meta/lib/patchtest/tests/test_mbox_cve.py b/meta/lib/patchtest/tests/test_mbox_cve.py
index 36548aa10c..af3712c192 100644
--- a/meta/lib/patchtest/tests/test_mbox_cve.py
+++ b/meta/lib/patchtest/tests/test_mbox_cve.py
@@ -20,12 +20,13 @@
 import base
 import os
 import parse_cve_tags
-import re
+import pyparsing
 
 class CVE(base.Base):
 
-    revert_shortlog_regex = re.compile('Revert\s+".*"')
+    revert_shortlog_regex = pyparsing.Regex('Revert\s+".*"')
     prog = parse_cve_tags.cve_tag
+    patch_prog = parse_cve_tags.patch_cve_tag
 
     def setUp(self):
         if self.unidiff_parse_error:
@@ -34,15 +35,17 @@ class CVE(base.Base):
         # we are just interested in series that introduce CVE patches, thus discard other
         # possibilities: modification to current CVEs, patch directly introduced into the
         # recipe, upgrades already including the CVE, etc.
-        new_cves = [p for p in self.patchset if p.path.endswith('.patch') and p.is_added_file]
-        if not new_cves:
-            self.skip('No new CVE patches introduced')
+        new_patches = [p for p in self.patchset if p.path.endswith('.patch') and p.is_added_file]
+        if not new_patches:
+            self.skip('No new patches introduced')
 
     def test_cve_presence_in_commit_message(self):
         for commit in CVE.commits:
             # skip those patches that revert older commits, these do not required the tag presence
-            if self.revert_shortlog_regex.match(commit.shortlog):
+            if self.revert_shortlog_regex.search_string(commit.shortlog):
                 continue
-            if not self.prog.search_string(commit.payload):
+            if not self.patch_prog.search_string(commit.payload):
+                self.skip("No CVE tag in added patch, so not needed in mbox")
+            elif not self.prog.search_string(commit.payload):
                 self.fail('Missing or incorrectly formatted CVE tag in mbox. Correct or include the CVE tag in the mbox with format: "CVE: CVE-YYYY-XXXX"',
                           commit=commit)
diff --git a/meta/lib/patchtest/tests/test_mbox_format.py b/meta/lib/patchtest/tests/test_mbox_format.py
index 42a8491a09..c9e0465835 100644
--- a/meta/lib/patchtest/tests/test_mbox_format.py
+++ b/meta/lib/patchtest/tests/test_mbox_format.py
@@ -5,11 +5,10 @@
 # SPDX-License-Identifier: GPL-2.0
 
 import base
-import re
 
 class MboxFormat(base.Base):
 
     def test_mbox_format(self):
         if self.unidiff_parse_error:
             self.fail('Series cannot be parsed correctly due to malformed diff lines. Create the series again using git-format-patch and ensure it can be applied using git am',
-                      data=[('Diff line', re.sub('^.+:\s(?<!$)','',self.unidiff_parse_error))])
+                      data=[('Diff line',self.unidiff_parse_error)])
diff --git a/meta/lib/patchtest/tests/test_mbox_mailinglist.py b/meta/lib/patchtest/tests/test_mbox_mailinglist.py
index 1f9e0be07f..c02c71ac6a 100644
--- a/meta/lib/patchtest/tests/test_mbox_mailinglist.py
+++ b/meta/lib/patchtest/tests/test_mbox_mailinglist.py
@@ -7,7 +7,7 @@
 import subprocess
 import collections
 import base
-import re
+import pyparsing
 from data import PatchTestInput
 
 class MailingList(base.Base):
@@ -39,9 +39,9 @@ class MailingList(base.Base):
 
         # a meta project may be indicted in the message subject, if this is the case, just fail
         # TODO: there may be other project with no-meta prefix, we also need to detect these
-        project_regex = re.compile("\[(?P<project>meta-.+)\]")
+        project_regex = pyparsing.Regex("\[(?P<project>meta-.+)\]")
         for commit in MailingList.commits:
-            match = project_regex.match(commit.subject)
+            match = project_regex.search_string(commit.subject)
             if match:
                 self.fail('Series sent to the wrong mailing list. Check the project\'s README (%s) and send the patch to the indicated list' % match.group('project'),
                           commit=commit)
diff --git a/meta/lib/patchtest/tests/test_mbox_signed_off_by.py b/meta/lib/patchtest/tests/test_mbox_signed_off_by.py
index 8fd705a2ef..59a89bd1bc 100644
--- a/meta/lib/patchtest/tests/test_mbox_signed_off_by.py
+++ b/meta/lib/patchtest/tests/test_mbox_signed_off_by.py
@@ -6,11 +6,11 @@
 
 import base
 import parse_signed_off_by
-import re
+import pyparsing
 
 class SignedOffBy(base.Base):
 
-    revert_shortlog_regex = re.compile('Revert\s+".*"')
+    revert_shortlog_regex = pyparsing.Regex('Revert\s+".*"')
 
     @classmethod
     def setUpClassLocal(cls):
@@ -20,7 +20,7 @@ class SignedOffBy(base.Base):
     def test_signed_off_by_presence(self):
         for commit in SignedOffBy.commits:
             # skip those patches that revert older commits, these do not required the tag presence
-            if self.revert_shortlog_regex.match(commit.shortlog):
+            if self.revert_shortlog_regex.search_string(commit.shortlog):
                 continue
             if not SignedOffBy.prog.search_string(commit.payload):
                 self.fail('Mbox is missing Signed-off-by. Add it manually or with "git commit --amend -s"',
diff --git a/meta/lib/patchtest/tests/test_metadata_lic_files_chksum.py b/meta/lib/patchtest/tests/test_metadata_lic_files_chksum.py
index a25a65c6db..784d432f01 100644
--- a/meta/lib/patchtest/tests/test_metadata_lic_files_chksum.py
+++ b/meta/lib/patchtest/tests/test_metadata_lic_files_chksum.py
@@ -5,7 +5,7 @@
 # SPDX-License-Identifier: GPL-2.0
 
 import base
-import re
+import pyparsing
 from data import PatchTestInput, PatchTestDataStore
 
 class LicFilesChkSum(base.Metadata):
@@ -13,7 +13,7 @@ class LicFilesChkSum(base.Metadata):
     license  = 'LICENSE'
     closed   = 'CLOSED'
     lictag   = 'License-Update'
-    lictag_re  = re.compile("^%s:" % lictag, re.MULTILINE)
+    lictag_re  = pyparsing.Regex("^%s:" % lictag)
 
     def setUp(self):
         # these tests just make sense on patches that can be merged
@@ -73,7 +73,7 @@ class LicFilesChkSum(base.Metadata):
             if pretest != test:
                 # if any patch on the series contain reference on the metadata, fail
                 for commit in self.commits:
-                    if self.lictag_re.search(commit.commit_message):
+                    if self.lictag_re.search_string(commit.commit_message):
                        break
                 else:
                     self.fail('LIC_FILES_CHKSUM changed on target %s but there is no "%s" tag in commit message. Include it with a brief description' % (pn, self.lictag),
diff --git a/meta/lib/patchtest/tests/test_metadata_max_length.py b/meta/lib/patchtest/tests/test_metadata_max_length.py
index b3a5dc9b79..477a9bff57 100644
--- a/meta/lib/patchtest/tests/test_metadata_max_length.py
+++ b/meta/lib/patchtest/tests/test_metadata_max_length.py
@@ -5,10 +5,10 @@
 # SPDX-License-Identifier: GPL-2.0
 
 import base
-import re
+import pyparsing
 
 class MaxLength(base.Base):
-    add_mark = re.compile('\+ ')
+    add_mark = pyparsing.Regex('\+ ')
     max_length = 200
 
     def test_max_line_length(self):
@@ -18,7 +18,7 @@ class MaxLength(base.Base):
                 continue
             payload = str(patch)
             for line in payload.splitlines():
-                if self.add_mark.match(line):
+                if self.add_mark.search_string(line):
                     current_line_length = len(line[1:])
                     if current_line_length > self.max_length:
                         self.fail('Patch line too long (current length %s, maximum is %s)' % (current_line_length, self.max_length),
diff --git a/meta/lib/patchtest/tests/test_metadata_src_uri.py b/meta/lib/patchtest/tests/test_metadata_src_uri.py
index ce2ace17bb..c19582ec2d 100644
--- a/meta/lib/patchtest/tests/test_metadata_src_uri.py
+++ b/meta/lib/patchtest/tests/test_metadata_src_uri.py
@@ -6,8 +6,8 @@
 
 import subprocess
 import base
-import re
 import os
+import pyparsing
 from data import PatchTestInput, PatchTestDataStore
 
 class SrcUri(base.Metadata):
@@ -15,7 +15,7 @@ class SrcUri(base.Metadata):
     metadata  = 'SRC_URI'
     md5sum    = 'md5sum'
     sha256sum = 'sha256sum'
-    git_regex = re.compile('^git\:\/\/.*')
+    git_regex = pyparsing.Regex('^git\:\/\/.*')
 
     def setUp(self):
         # these tests just make sense on patches that can be merged
diff --git a/meta/lib/patchtest/tests/test_patch_cve.py b/meta/lib/patchtest/tests/test_patch_cve.py
index 144e130707..0ae85adcf9 100644
--- a/meta/lib/patchtest/tests/test_patch_cve.py
+++ b/meta/lib/patchtest/tests/test_patch_cve.py
@@ -19,12 +19,12 @@
 
 import base
 import os
-import re
+import pyparsing
 
 class CVE(base.Base):
 
-    re_cve_pattern = re.compile("CVE\-\d{4}\-\d+", re.IGNORECASE)
-    re_cve_payload_tag     = re.compile("\+CVE:(\s+CVE\-\d{4}\-\d+)+")
+    re_cve_pattern = pyparsing.Regex("CVE\-\d{4}\-\d+")
+    re_cve_payload_tag = pyparsing.Regex("\+CVE:(\s+CVE\-\d{4}\-\d+)+")
 
     def setUp(self):
         if self.unidiff_parse_error:
@@ -39,10 +39,10 @@ class CVE(base.Base):
 
     def test_cve_tag_format(self):
         for commit in CVE.commits:
-            if self.re_cve_pattern.search(commit.shortlog) or self.re_cve_pattern.search(commit.commit_message):
+            if self.re_cve_pattern.search_string(commit.shortlog) or self.re_cve_pattern.search_string(commit.commit_message):
                 tag_found = False
                 for line in commit.payload.splitlines():
-                    if self.re_cve_payload_tag.match(line):
+                    if self.re_cve_payload_tag.search_string(line):
                         tag_found = True
                         break
                 if not tag_found:
diff --git a/meta/lib/patchtest/tests/test_patch_signed_off_by.py b/meta/lib/patchtest/tests/test_patch_signed_off_by.py
index 5892033af0..2df6419d26 100644
--- a/meta/lib/patchtest/tests/test_patch_signed_off_by.py
+++ b/meta/lib/patchtest/tests/test_patch_signed_off_by.py
@@ -6,7 +6,6 @@
 
 import base
 import parse_signed_off_by
-import re
 
 class PatchSignedOffBy(base.Base):