From ec065ce64c1d0155e8566c7dd12911ae0c7a04d0 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Fri, 22 Aug 2025 00:40:37 +0200 Subject: bitbake: Use a "fork" multiprocessing context Python 3.14 changes the default multiprocessing context from "fork" to "forkserver"; however bitbake heavily relies on "fork" to efficiently pass data to the child processes. As such, make "fork" context in the bb namespace and use it in place of the normal multiprocessing module. Note that multiprocessing contexts were added in Python 3.4, so this should be safe to use even before Python 3.14 [YOCTO #15858] (Bitbake rev: 0d9d8d0863e82e986c33c08064ce9a99224a06d5) Signed-off-by: Joshua Watt Signed-off-by: Richard Purdie Signed-off-by: Steve Sakoman --- bitbake/lib/bb/__init__.py | 28 ++++++++++++++++++++++++++++ bitbake/lib/bb/asyncrpc/serv.py | 2 +- bitbake/lib/bb/cooker.py | 2 +- bitbake/lib/bb/server/process.py | 2 +- bitbake/lib/bb/tests/support/httpserver.py | 4 ++-- bitbake/lib/bb/utils.py | 4 +--- bitbake/lib/hashserv/tests.py | 2 +- 7 files changed, 35 insertions(+), 9 deletions(-) diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py index 5668355737..605355e1ad 100644 --- a/bitbake/lib/bb/__init__.py +++ b/bitbake/lib/bb/__init__.py @@ -37,6 +37,34 @@ class BBHandledException(Exception): import os import logging from collections import namedtuple +import multiprocessing as mp + +# Python 3.14 changes the default multiprocessing context from "fork" to +# "forkserver". However, bitbake heavily relies on "fork" behavior to +# efficiently pass data to the child processes. Places that need this should do: +# from bb import multiprocessing +# in place of +# import multiprocessing + +class MultiprocessingContext(object): + """ + Multiprocessing proxy object that uses the "fork" context for a property if + available, otherwise goes to the main multiprocessing module. This allows + it to be a drop-in replacement for the multiprocessing module, but use the + fork context + """ + def __init__(self): + super().__setattr__("_ctx", mp.get_context("fork")) + + def __getattr__(self, name): + if hasattr(self._ctx, name): + return getattr(self._ctx, name) + return getattr(mp, name) + + def __setattr__(self, name, value): + raise AttributeError(f"Unable to set attribute {name}") + +multiprocessing = MultiprocessingContext() class NullHandler(logging.Handler): diff --git a/bitbake/lib/bb/asyncrpc/serv.py b/bitbake/lib/bb/asyncrpc/serv.py index 667217c5c1..acdcaf08ed 100644 --- a/bitbake/lib/bb/asyncrpc/serv.py +++ b/bitbake/lib/bb/asyncrpc/serv.py @@ -11,7 +11,7 @@ import os import signal import socket import sys -import multiprocessing +from bb import multiprocessing import logging from .connection import StreamConnection, WebsocketConnection from .exceptions import ClientError, ServerError, ConnectionClosedError, InvokeError diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 1810bcc604..36659cee26 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py @@ -12,7 +12,7 @@ import enum import sys, os, glob, os.path, re, time import itertools import logging -import multiprocessing +from bb import multiprocessing import threading from io import StringIO, UnsupportedOperation from contextlib import closing diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py index 4b35be62cd..f74b3f1bf2 100644 --- a/bitbake/lib/bb/server/process.py +++ b/bitbake/lib/bb/server/process.py @@ -13,7 +13,7 @@ import bb import bb.event import logging -import multiprocessing +from bb import multiprocessing import threading import array import os diff --git a/bitbake/lib/bb/tests/support/httpserver.py b/bitbake/lib/bb/tests/support/httpserver.py index 78f7660053..03327e923b 100644 --- a/bitbake/lib/bb/tests/support/httpserver.py +++ b/bitbake/lib/bb/tests/support/httpserver.py @@ -3,7 +3,7 @@ # import http.server -import multiprocessing +from bb import multiprocessing import os import traceback import signal @@ -43,7 +43,7 @@ class HTTPService(object): self.process = multiprocessing.Process(target=self.server.server_start, args=[self.root_dir, self.logger]) # The signal handler from testimage.bbclass can cause deadlocks here - # if the HTTPServer is terminated before it can restore the standard + # if the HTTPServer is terminated before it can restore the standard #signal behaviour orig = signal.getsignal(signal.SIGTERM) signal.signal(signal.SIGTERM, signal.SIG_DFL) diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py index 0832422683..694e79f552 100644 --- a/bitbake/lib/bb/utils.py +++ b/bitbake/lib/bb/utils.py @@ -12,7 +12,7 @@ import sys import errno import logging import locale -import multiprocessing +from bb import multiprocessing import importlib import importlib.machinery import importlib.util @@ -1198,8 +1198,6 @@ def process_profilelog(fn, pout = None): # def multiprocessingpool(*args, **kwargs): - import multiprocessing.pool - #import multiprocessing.util #multiprocessing.util.log_to_stderr(10) # Deal with a multiprocessing bug where signals to the processes would be delayed until the work # completes. Putting in a timeout means the signals (like SIGINT/SIGTERM) get processed. diff --git a/bitbake/lib/hashserv/tests.py b/bitbake/lib/hashserv/tests.py index da3f8e0884..124d8aa005 100644 --- a/bitbake/lib/hashserv/tests.py +++ b/bitbake/lib/hashserv/tests.py @@ -10,7 +10,7 @@ from .server import DEFAULT_ANON_PERMS, ALL_PERMISSIONS from bb.asyncrpc import InvokeError import hashlib import logging -import multiprocessing +from bb import multiprocessing import os import sys import tempfile -- cgit v1.2.3-54-g00ecf