qemurunner.py: refactor searching for QEMU PID

Ask QEMU to store the QEMU process PID in a file, this way we don't
have to parse running processes and analyze descendents.
This is done via QEMU command line argument "-pidfile".

[YOCTO #12001]

(From OE-Core rev: 67612dcd2a8a1aa1d683dddb0bd2f592886ff020)

(From OE-Core rev: 9fe7aa1824666d3f7ceb085bef59026e245cb752)

Signed-off-by: Juro Bystricky <juro.bystricky@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Armin Kuster <akuster808@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 136 insertions, 167 deletions

diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py
index 3148d7ec22..59886b10ed 100644
--- a/meta/lib/oeqa/utils/qemurunner.py
+++ b/meta/lib/oeqa/utils/qemurunner.py
@@ -53,6 +53,7 @@ class QemuRunner:
         self.use_kvm = use_kvm
 
         self.runqemutime = 120
+        self.qemu_pidfile = 'pidfile_'+str(os.getpid())
         self.host_dumper = HostDumper(dump_host_cmds, dump_dir)
 
         self.logger = logging.getLogger("BitBake.QemuRunner")
@@ -142,7 +143,11 @@ class QemuRunner:
         if extra_bootparams:
             bootparams = bootparams + ' ' + extra_bootparams
 
-        self.qemuparams = 'bootparams="{0}" qemuparams="-serial tcp:127.0.0.1:{1}"'.format(bootparams, threadport)
+        # Ask QEMU to store the QEMU process PID in file, this way we don't have to parse running processes
+        # and analyze descendents in order to determine it.
+        if os.path.exists(self.qemu_pidfile):
+            os.remove(self.qemu_pidfile)
+        self.qemuparams = 'bootparams="{0}" qemuparams="-serial tcp:127.0.0.1:{1} -pidfile {2}"'.format(bootparams, threadport, self.qemu_pidfile)
         if qemuparams:
             self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"'
 
@@ -198,125 +203,14 @@ class QemuRunner:
                     self.stop()
                     self.logger.info("Output from runqemu:\n%s" % self.getOutput(output))
                     return False
-            time.sleep(1)
+            time.sleep(0.5)
 
-        out = self.getOutput(output)
-        netconf = False # network configuration is not required by default
-        if self.is_alive():
-            self.logger.info("qemu started in %s seconds - qemu procces pid is %s" % (time.time() - (endtime - self.runqemutime), self.qemupid))
-            if get_ip:
-                cmdline = ''
-                with open('/proc/%s/cmdline' % self.qemupid) as p:
-                    cmdline = p.read()
-                    # It is needed to sanitize the data received
-                    # because is possible to have control characters
-                    cmdline = re_control_char.sub('', cmdline)
-                try:
-                    ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1])
-                    self.ip = ips[0]
-                    self.server_ip = ips[1]
-                    self.logger.info("qemu cmdline used:\n{}".format(cmdline))
-                except (IndexError, ValueError):
-                    # Try to get network configuration from runqemu output
-                    match = re.match('.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*',
-                                     out, re.MULTILINE|re.DOTALL)
-                    if match:
-                        self.ip, self.server_ip, self.netmask = match.groups()
-                        # network configuration is required as we couldn't get it
-                        # from the runqemu command line, so qemu doesn't run kernel
-                        # and guest networking is not configured
-                        netconf = True
-                    else:
-                        self.logger.error("Couldn't get ip from qemu command line and runqemu output! "
-                                     "Here is the qemu command line used:\n%s\n"
-                                     "and output from runqemu:\n%s" % (cmdline, out))
-                        self._dump_host()
-                        self.stop()
-                        return False
-
-                self.logger.info("Target IP: %s" % self.ip)
-                self.logger.info("Server IP: %s" % self.server_ip)
-
-            self.thread = LoggingThread(self.log, threadsock, self.logger)
-            self.thread.start()
-            if not self.thread.connection_established.wait(self.boottime):
-                self.logger.error("Didn't receive a console connection from qemu. "
-                             "Here is the qemu command line used:\n%s\nand "
-                             "output from runqemu:\n%s" % (cmdline, out))
-                self.stop_thread()
-                return False
-
-            self.logger.info("Output from runqemu:\n%s", out)
-            self.logger.info("Waiting at most %d seconds for login banner" % self.boottime)
-            endtime = time.time() + self.boottime
-            socklist = [self.server_socket]
-            reachedlogin = False
-            stopread = False
-            qemusock = None
-            bootlog = ''
-            data = b''
-            while time.time() < endtime and not stopread:
-                try:
-                    sread, swrite, serror = select.select(socklist, [], [], 5)
-                except InterruptedError:
-                    continue
-                for sock in sread:
-                    if sock is self.server_socket:
-                        qemusock, addr = self.server_socket.accept()
-                        qemusock.setblocking(0)
-                        socklist.append(qemusock)
-                        socklist.remove(self.server_socket)
-                        self.logger.info("Connection from %s:%s" % addr)
-                    else:
-                        data = data + sock.recv(1024)
-                        if data:
-                            try:
-                                data = data.decode("utf-8", errors="surrogateescape")
-                                bootlog += data
-                                data = b''
-                                if re.search(".* login:", bootlog):
-                                    self.server_socket = qemusock
-                                    stopread = True
-                                    reachedlogin = True
-                                    self.logger.info("Reached login banner")
-                            except UnicodeDecodeError:
-                                continue
-                        else:
-                            socklist.remove(sock)
-                            sock.close()
-                            stopread = True
-
-            if not reachedlogin:
-                self.logger.info("Target didn't reached login boot in %d seconds" % self.boottime)
-                lines = "\n".join(bootlog.splitlines()[-25:])
-                self.logger.info("Last 25 lines of text:\n%s" % lines)
-                self.logger.info("Check full boot log: %s" % self.logfile)
-                self._dump_host()
-                self.stop()
-                return False
-
-            # If we are not able to login the tests can continue
-            try:
-                (status, output) = self.run_serial("root\n", raw=True)
-                if re.search("root@[a-zA-Z0-9\-]+:~#", output):
-                    self.logged = True
-                    self.logger.info("Logged as root in serial console")
-                    if netconf:
-                        # configure guest networking
-                        cmd = "ifconfig eth0 %s netmask %s up\n" % (self.ip, self.netmask)
-                        output = self.run_serial(cmd, raw=True)[1]
-                        if re.search("root@[a-zA-Z0-9\-]+:~#", output):
-                            self.logger.info("configured ip address %s", self.ip)
-                        else:
-                            self.logger.info("Couldn't configure guest networking")
-                else:
-                    self.logger.info("Couldn't login into serial console"
-                            " as root using blank password")
-            except:
-                self.logger.info("Serial console failed while trying to login")
-
-        else:
+        if not self.is_alive():
             self.logger.error("Qemu pid didn't appear in %s seconds" % self.runqemutime)
+            # Dump all processes to help us to figure out what is going on...
+            ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command '], stdout=subprocess.PIPE).communicate()[0]
+            processes = ps.decode("utf-8")
+            self.logger.info("Running processes:\n%s" % processes)
             self._dump_host()
             self.stop()
             op = self.getOutput(output)
@@ -326,7 +220,121 @@ class QemuRunner:
                 self.logger.error("No output from runqemu.\n")
             return False
 
-        return self.is_alive()
+        # We are alive: qemu is running
+        out = self.getOutput(output)
+        netconf = False # network configuration is not required by default
+        self.logger.info("qemu started in %s seconds - qemu procces pid is %s" % (time.time() - (endtime - self.runqemutime), self.qemupid))
+        if get_ip:
+            cmdline = ''
+            with open('/proc/%s/cmdline' % self.qemupid) as p:
+                cmdline = p.read()
+                # It is needed to sanitize the data received
+                # because is possible to have control characters
+                cmdline = re_control_char.sub(' ', cmdline)
+            try:
+                ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1])
+                self.ip = ips[0]
+                self.server_ip = ips[1]
+                self.logger.info("qemu cmdline used:\n{}".format(cmdline))
+            except (IndexError, ValueError):
+                # Try to get network configuration from runqemu output
+                match = re.match('.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*',
+                                 out, re.MULTILINE|re.DOTALL)
+                if match:
+                    self.ip, self.server_ip, self.netmask = match.groups()
+                    # network configuration is required as we couldn't get it
+                    # from the runqemu command line, so qemu doesn't run kernel
+                    # and guest networking is not configured
+                    netconf = True
+                else:
+                    self.logger.error("Couldn't get ip from qemu command line and runqemu output! "
+                                 "Here is the qemu command line used:\n%s\n"
+                                 "and output from runqemu:\n%s" % (cmdline, out))
+                    self._dump_host()
+                    self.stop()
+                    return False
+
+        self.logger.info("Target IP: %s" % self.ip)
+        self.logger.info("Server IP: %s" % self.server_ip)
+
+        self.thread = LoggingThread(self.log, threadsock, self.logger)
+        self.thread.start()
+        if not self.thread.connection_established.wait(self.boottime):
+            self.logger.error("Didn't receive a console connection from qemu. "
+                         "Here is the qemu command line used:\n%s\nand "
+                         "output from runqemu:\n%s" % (cmdline, out))
+            self.stop_thread()
+            return False
+
+        self.logger.info("Output from runqemu:\n%s", out)
+        self.logger.info("Waiting at most %d seconds for login banner" % self.boottime)
+        endtime = time.time() + self.boottime
+        socklist = [self.server_socket]
+        reachedlogin = False
+        stopread = False
+        qemusock = None
+        bootlog = ''
+        data = b''
+        while time.time() < endtime and not stopread:
+            try:
+                sread, swrite, serror = select.select(socklist, [], [], 5)
+            except InterruptedError:
+                continue
+            for sock in sread:
+                if sock is self.server_socket:
+                    qemusock, addr = self.server_socket.accept()
+                    qemusock.setblocking(0)
+                    socklist.append(qemusock)
+                    socklist.remove(self.server_socket)
+                    self.logger.info("Connection from %s:%s" % addr)
+                else:
+                    data = data + sock.recv(1024)
+                    if data:
+                        try:
+                            data = data.decode("utf-8", errors="surrogateescape")
+                            bootlog += data
+                            data = b''
+                            if re.search(".* login:", bootlog):
+                                self.server_socket = qemusock
+                                stopread = True
+                                reachedlogin = True
+                                self.logger.info("Reached login banner")
+                        except UnicodeDecodeError:
+                            continue
+                    else:
+                        socklist.remove(sock)
+                        sock.close()
+                        stopread = True
+
+        if not reachedlogin:
+            self.logger.info("Target didn't reached login boot in %d seconds" % self.boottime)
+            lines = "\n".join(bootlog.splitlines()[-25:])
+            self.logger.info("Last 25 lines of text:\n%s" % lines)
+            self.logger.info("Check full boot log: %s" % self.logfile)
+            self._dump_host()
+            self.stop()
+            return False
+
+        # If we are not able to login the tests can continue
+        try:
+            (status, output) = self.run_serial("root\n", raw=True)
+            if re.search("root@[a-zA-Z0-9\-]+:~#", output):
+                self.logged = True
+                self.logger.info("Logged as root in serial console")
+                if netconf:
+                    # configure guest networking
+                    cmd = "ifconfig eth0 %s netmask %s up\n" % (self.ip, self.netmask)
+                    output = self.run_serial(cmd, raw=True)[1]
+                    if re.search("root@[a-zA-Z0-9\-]+:~#", output):
+                        self.logger.info("configured ip address %s", self.ip)
+                    else:
+                        self.logger.info("Couldn't configure guest networking")
+                else:
+                    self.logger.info("Couldn't login into serial console"
+                                " as root using blank password")
+        except:
+            self.logger.info("Serial console failed while trying to login")
+        return True
 
     def stop(self):
         self.stop_thread()
@@ -354,6 +362,8 @@ class QemuRunner:
             self.server_socket = None
         self.qemupid = None
         self.ip = None
+        if os.path.exists(self.qemu_pidfile):
+            os.remove(self.qemu_pidfile)
 
     def stop_qemu_system(self):
         if self.qemupid:
@@ -379,56 +389,15 @@ class QemuRunner:
     def is_alive(self):
         if not self.runqemu:
             return False
-        qemu_child = self.find_child(str(self.runqemu.pid))
-        if qemu_child:
-            self.qemupid = qemu_child[0]
-            if os.path.exists("/proc/" + str(self.qemupid)):
-                return True
+        if os.path.isfile(self.qemu_pidfile):
+            f = open(self.qemu_pidfile, 'r')
+            qemu_pid = f.read()
+            f.close()
+            #logger.info("qemu_pid: %s" % qemu_pid)
+            self.qemupid = int(qemu_pid)
+            return True
         return False
 
-    def find_child(self,parent_pid):
-        #
-        # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd]
-        #
-        ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0]
-        processes = ps.decode("utf-8").split('\n')
-        nfields = len(processes[0].split()) - 1
-        pids = {}
-        commands = {}
-        for row in processes[1:]:
-            data = row.split(None, nfields)
-            if len(data) != 3:
-                continue
-            if data[1] not in pids:
-                pids[data[1]] = []
-
-            pids[data[1]].append(data[0])
-            commands[data[0]] = data[2]
-
-        if parent_pid not in pids:
-            return []
-
-        parents = []
-        newparents = pids[parent_pid]
-        while newparents:
-            next = []
-            for p in newparents:
-                if p in pids:
-                    for n in pids[p]:
-                        if n not in parents and n not in next:
-                            next.append(n)
-                if p not in parents:
-                    parents.append(p)
-                    newparents = next
-        #print("Children matching %s:" % str(parents))
-        for p in parents:
-            # Need to be careful here since runqemu runs "ldd qemu-system-xxxx"
-            # Also, old versions of ldd (2.11) run "LD_XXXX qemu-system-xxxx"
-            basecmd = commands[p].split()[0]
-            basecmd = os.path.basename(basecmd)
-            if "qemu-system" in basecmd and "-serial tcp" in commands[p]:
-                return [int(p),commands[p]]
-
     def run_serial(self, command, raw=False, timeout=5):
         # We assume target system have echo to get command status
         if not raw: