diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2021-12-13 14:24:40 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2021-12-14 22:47:04 +0000 |
commit | 7c0266a24892a35197e8ab603a16b78192dd589d (patch) | |
tree | cdc6d3e2a1a9bd9a2372d717b84cd0cc6075f71d /bitbake | |
parent | 5de2528a4ba08d3372ed47ad13ee0679d6b05761 (diff) | |
download | poky-7c0266a24892a35197e8ab603a16b78192dd589d.tar.gz |
bitbake: tests/runqueue: Improve lockfile handling race
Internal bitbake shutdown timings must have changed recently as we're
seeing race issues on the autobuilder around the removal of the bitbake.lock
file. Improve the lockfile race code to cover bitbake's lockfile too
and use it in all the tests.
[YOCTO #14658]
[YOCTO #14652]
(Bitbake rev: bd1912bed64424f9fb28396b71bb49b6090ed087)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rw-r--r-- | bitbake/lib/bb/tests/runqueue.py | 40 |
1 files changed, 38 insertions, 2 deletions
diff --git a/bitbake/lib/bb/tests/runqueue.py b/bitbake/lib/bb/tests/runqueue.py index 2bf00dc62a..35d5a843ff 100644 --- a/bitbake/lib/bb/tests/runqueue.py +++ b/bitbake/lib/bb/tests/runqueue.py | |||
@@ -59,6 +59,8 @@ class RunQueueTests(unittest.TestCase): | |||
59 | expected = ['a1:' + x for x in self.alltasks] | 59 | expected = ['a1:' + x for x in self.alltasks] |
60 | self.assertEqual(set(tasks), set(expected)) | 60 | self.assertEqual(set(tasks), set(expected)) |
61 | 61 | ||
62 | self.shutdown(tempdir) | ||
63 | |||
62 | def test_single_setscenevalid(self): | 64 | def test_single_setscenevalid(self): |
63 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 65 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
64 | cmd = ["bitbake", "a1"] | 66 | cmd = ["bitbake", "a1"] |
@@ -69,6 +71,8 @@ class RunQueueTests(unittest.TestCase): | |||
69 | 'a1:populate_sysroot', 'a1:build'] | 71 | 'a1:populate_sysroot', 'a1:build'] |
70 | self.assertEqual(set(tasks), set(expected)) | 72 | self.assertEqual(set(tasks), set(expected)) |
71 | 73 | ||
74 | self.shutdown(tempdir) | ||
75 | |||
72 | def test_intermediate_setscenevalid(self): | 76 | def test_intermediate_setscenevalid(self): |
73 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 77 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
74 | cmd = ["bitbake", "a1"] | 78 | cmd = ["bitbake", "a1"] |
@@ -78,6 +82,8 @@ class RunQueueTests(unittest.TestCase): | |||
78 | 'a1:populate_sysroot_setscene', 'a1:build'] | 82 | 'a1:populate_sysroot_setscene', 'a1:build'] |
79 | self.assertEqual(set(tasks), set(expected)) | 83 | self.assertEqual(set(tasks), set(expected)) |
80 | 84 | ||
85 | self.shutdown(tempdir) | ||
86 | |||
81 | def test_intermediate_notcovered(self): | 87 | def test_intermediate_notcovered(self): |
82 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 88 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
83 | cmd = ["bitbake", "a1"] | 89 | cmd = ["bitbake", "a1"] |
@@ -87,6 +93,8 @@ class RunQueueTests(unittest.TestCase): | |||
87 | 'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene'] | 93 | 'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene'] |
88 | self.assertEqual(set(tasks), set(expected)) | 94 | self.assertEqual(set(tasks), set(expected)) |
89 | 95 | ||
96 | self.shutdown(tempdir) | ||
97 | |||
90 | def test_all_setscenevalid(self): | 98 | def test_all_setscenevalid(self): |
91 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 99 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
92 | cmd = ["bitbake", "a1"] | 100 | cmd = ["bitbake", "a1"] |
@@ -96,6 +104,8 @@ class RunQueueTests(unittest.TestCase): | |||
96 | 'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene'] | 104 | 'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene'] |
97 | self.assertEqual(set(tasks), set(expected)) | 105 | self.assertEqual(set(tasks), set(expected)) |
98 | 106 | ||
107 | self.shutdown(tempdir) | ||
108 | |||
99 | def test_no_settasks(self): | 109 | def test_no_settasks(self): |
100 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 110 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
101 | cmd = ["bitbake", "a1", "-c", "patch"] | 111 | cmd = ["bitbake", "a1", "-c", "patch"] |
@@ -104,6 +114,8 @@ class RunQueueTests(unittest.TestCase): | |||
104 | expected = ['a1:fetch', 'a1:unpack', 'a1:patch'] | 114 | expected = ['a1:fetch', 'a1:unpack', 'a1:patch'] |
105 | self.assertEqual(set(tasks), set(expected)) | 115 | self.assertEqual(set(tasks), set(expected)) |
106 | 116 | ||
117 | self.shutdown(tempdir) | ||
118 | |||
107 | def test_mix_covered_notcovered(self): | 119 | def test_mix_covered_notcovered(self): |
108 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 120 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
109 | cmd = ["bitbake", "a1:do_patch", "a1:do_populate_sysroot"] | 121 | cmd = ["bitbake", "a1:do_patch", "a1:do_populate_sysroot"] |
@@ -112,6 +124,7 @@ class RunQueueTests(unittest.TestCase): | |||
112 | expected = ['a1:fetch', 'a1:unpack', 'a1:patch', 'a1:populate_sysroot_setscene'] | 124 | expected = ['a1:fetch', 'a1:unpack', 'a1:patch', 'a1:populate_sysroot_setscene'] |
113 | self.assertEqual(set(tasks), set(expected)) | 125 | self.assertEqual(set(tasks), set(expected)) |
114 | 126 | ||
127 | self.shutdown(tempdir) | ||
115 | 128 | ||
116 | # Test targets with intermediate setscene tasks alongside a target with no intermediate setscene tasks | 129 | # Test targets with intermediate setscene tasks alongside a target with no intermediate setscene tasks |
117 | def test_mixed_direct_tasks_setscene_tasks(self): | 130 | def test_mixed_direct_tasks_setscene_tasks(self): |
@@ -123,6 +136,8 @@ class RunQueueTests(unittest.TestCase): | |||
123 | 'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene'] | 136 | 'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene'] |
124 | self.assertEqual(set(tasks), set(expected)) | 137 | self.assertEqual(set(tasks), set(expected)) |
125 | 138 | ||
139 | self.shutdown(tempdir) | ||
140 | |||
126 | # This test slows down the execution of do_package_setscene until after other real tasks have | 141 | # This test slows down the execution of do_package_setscene until after other real tasks have |
127 | # started running which tests for a bug where tasks were being lost from the buildable list of real | 142 | # started running which tests for a bug where tasks were being lost from the buildable list of real |
128 | # tasks if they weren't in tasks_covered or tasks_notcovered | 143 | # tasks if they weren't in tasks_covered or tasks_notcovered |
@@ -137,6 +152,8 @@ class RunQueueTests(unittest.TestCase): | |||
137 | 'a1:populate_sysroot', 'a1:build'] | 152 | 'a1:populate_sysroot', 'a1:build'] |
138 | self.assertEqual(set(tasks), set(expected)) | 153 | self.assertEqual(set(tasks), set(expected)) |
139 | 154 | ||
155 | self.shutdown(tempdir) | ||
156 | |||
140 | def test_setscenewhitelist(self): | 157 | def test_setscenewhitelist(self): |
141 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 158 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
142 | cmd = ["bitbake", "a1"] | 159 | cmd = ["bitbake", "a1"] |
@@ -150,6 +167,8 @@ class RunQueueTests(unittest.TestCase): | |||
150 | 'a1:populate_sysroot_setscene', 'a1:package_setscene'] | 167 | 'a1:populate_sysroot_setscene', 'a1:package_setscene'] |
151 | self.assertEqual(set(tasks), set(expected)) | 168 | self.assertEqual(set(tasks), set(expected)) |
152 | 169 | ||
170 | self.shutdown(tempdir) | ||
171 | |||
153 | # Tests for problems with dependencies between setscene tasks | 172 | # Tests for problems with dependencies between setscene tasks |
154 | def test_no_setscenevalid_harddeps(self): | 173 | def test_no_setscenevalid_harddeps(self): |
155 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 174 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
@@ -163,6 +182,8 @@ class RunQueueTests(unittest.TestCase): | |||
163 | 'd1:populate_sysroot', 'd1:build'] | 182 | 'd1:populate_sysroot', 'd1:build'] |
164 | self.assertEqual(set(tasks), set(expected)) | 183 | self.assertEqual(set(tasks), set(expected)) |
165 | 184 | ||
185 | self.shutdown(tempdir) | ||
186 | |||
166 | def test_no_setscenevalid_withdeps(self): | 187 | def test_no_setscenevalid_withdeps(self): |
167 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 188 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
168 | cmd = ["bitbake", "b1"] | 189 | cmd = ["bitbake", "b1"] |
@@ -173,6 +194,8 @@ class RunQueueTests(unittest.TestCase): | |||
173 | expected.remove('a1:package_qa') | 194 | expected.remove('a1:package_qa') |
174 | self.assertEqual(set(tasks), set(expected)) | 195 | self.assertEqual(set(tasks), set(expected)) |
175 | 196 | ||
197 | self.shutdown(tempdir) | ||
198 | |||
176 | def test_single_a1_setscenevalid_withdeps(self): | 199 | def test_single_a1_setscenevalid_withdeps(self): |
177 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 200 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
178 | cmd = ["bitbake", "b1"] | 201 | cmd = ["bitbake", "b1"] |
@@ -183,6 +206,8 @@ class RunQueueTests(unittest.TestCase): | |||
183 | 'a1:populate_sysroot'] + ['b1:' + x for x in self.alltasks] | 206 | 'a1:populate_sysroot'] + ['b1:' + x for x in self.alltasks] |
184 | self.assertEqual(set(tasks), set(expected)) | 207 | self.assertEqual(set(tasks), set(expected)) |
185 | 208 | ||
209 | self.shutdown(tempdir) | ||
210 | |||
186 | def test_single_b1_setscenevalid_withdeps(self): | 211 | def test_single_b1_setscenevalid_withdeps(self): |
187 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 212 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
188 | cmd = ["bitbake", "b1"] | 213 | cmd = ["bitbake", "b1"] |
@@ -194,6 +219,8 @@ class RunQueueTests(unittest.TestCase): | |||
194 | expected.remove('b1:package') | 219 | expected.remove('b1:package') |
195 | self.assertEqual(set(tasks), set(expected)) | 220 | self.assertEqual(set(tasks), set(expected)) |
196 | 221 | ||
222 | self.shutdown(tempdir) | ||
223 | |||
197 | def test_intermediate_setscenevalid_withdeps(self): | 224 | def test_intermediate_setscenevalid_withdeps(self): |
198 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 225 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
199 | cmd = ["bitbake", "b1"] | 226 | cmd = ["bitbake", "b1"] |
@@ -204,6 +231,8 @@ class RunQueueTests(unittest.TestCase): | |||
204 | expected.remove('b1:package') | 231 | expected.remove('b1:package') |
205 | self.assertEqual(set(tasks), set(expected)) | 232 | self.assertEqual(set(tasks), set(expected)) |
206 | 233 | ||
234 | self.shutdown(tempdir) | ||
235 | |||
207 | def test_all_setscenevalid_withdeps(self): | 236 | def test_all_setscenevalid_withdeps(self): |
208 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 237 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
209 | cmd = ["bitbake", "b1"] | 238 | cmd = ["bitbake", "b1"] |
@@ -214,6 +243,8 @@ class RunQueueTests(unittest.TestCase): | |||
214 | 'b1:packagedata_setscene', 'b1:package_qa_setscene', 'b1:populate_sysroot_setscene'] | 243 | 'b1:packagedata_setscene', 'b1:package_qa_setscene', 'b1:populate_sysroot_setscene'] |
215 | self.assertEqual(set(tasks), set(expected)) | 244 | self.assertEqual(set(tasks), set(expected)) |
216 | 245 | ||
246 | self.shutdown(tempdir) | ||
247 | |||
217 | def test_multiconfig_setscene_optimise(self): | 248 | def test_multiconfig_setscene_optimise(self): |
218 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 249 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
219 | extraenv = { | 250 | extraenv = { |
@@ -233,6 +264,8 @@ class RunQueueTests(unittest.TestCase): | |||
233 | expected.remove(x) | 264 | expected.remove(x) |
234 | self.assertEqual(set(tasks), set(expected)) | 265 | self.assertEqual(set(tasks), set(expected)) |
235 | 266 | ||
267 | self.shutdown(tempdir) | ||
268 | |||
236 | def test_multiconfig_bbmask(self): | 269 | def test_multiconfig_bbmask(self): |
237 | # This test validates that multiconfigs can independently mask off | 270 | # This test validates that multiconfigs can independently mask off |
238 | # recipes they do not want with BBMASK. It works by having recipes | 271 | # recipes they do not want with BBMASK. It works by having recipes |
@@ -249,6 +282,8 @@ class RunQueueTests(unittest.TestCase): | |||
249 | cmd = ["bitbake", "mc:mc-1:fails-mc2", "mc:mc_2:fails-mc1"] | 282 | cmd = ["bitbake", "mc:mc-1:fails-mc2", "mc:mc_2:fails-mc1"] |
250 | self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv) | 283 | self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv) |
251 | 284 | ||
285 | self.shutdown(tempdir) | ||
286 | |||
252 | def test_multiconfig_mcdepends(self): | 287 | def test_multiconfig_mcdepends(self): |
253 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 288 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
254 | extraenv = { | 289 | extraenv = { |
@@ -279,6 +314,8 @@ class RunQueueTests(unittest.TestCase): | |||
279 | ["mc_2:a1:%s" % t for t in rerun_tasks] | 314 | ["mc_2:a1:%s" % t for t in rerun_tasks] |
280 | self.assertEqual(set(tasks), set(expected)) | 315 | self.assertEqual(set(tasks), set(expected)) |
281 | 316 | ||
317 | self.shutdown(tempdir) | ||
318 | |||
282 | def test_hashserv_single(self): | 319 | def test_hashserv_single(self): |
283 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: | 320 | with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: |
284 | extraenv = { | 321 | extraenv = { |
@@ -359,7 +396,6 @@ class RunQueueTests(unittest.TestCase): | |||
359 | 396 | ||
360 | def shutdown(self, tempdir): | 397 | def shutdown(self, tempdir): |
361 | # Wait for the hashserve socket to disappear else we'll see races with the tempdir cleanup | 398 | # Wait for the hashserve socket to disappear else we'll see races with the tempdir cleanup |
362 | while (os.path.exists(tempdir + "/hashserve.sock") or os.path.exists(tempdir + "cache/hashserv.db-wal")): | 399 | while (os.path.exists(tempdir + "/hashserve.sock") or os.path.exists(tempdir + "cache/hashserv.db-wal") or os.path.exists(tempdir + "/bitbake.lock")): |
363 | time.sleep(0.5) | 400 | time.sleep(0.5) |
364 | 401 | ||
365 | |||