bitbake: bitbake: Convert to python 3

Various misc changes to convert bitbake to python3 which don't warrant
separation into separate commits.

(Bitbake rev: d0f904d407f57998419bd9c305ce53e5eaa36b24)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

10 files changed, 51 insertions, 43 deletions

diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake
index bba87b082c..9813a08483 100755
--- a/bitbake/bin/bitbake
+++ b/bitbake/bin/bitbake
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # ex:ts=4:sw=4:sts=4:et
 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
@@ -35,6 +35,9 @@ except RuntimeError as exc:
 from bb import cookerdata
 from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 
+if sys.getfilesystemencoding() != "utf-8":
+    sys.exit("Please use a locale setting which supports utf-8.\nPython can't change the filesystem locale after loading so we need a utf-8 when python starts or things won't work.")
+
 __version__ = "1.31.0"
 
 if __name__ == "__main__":
diff --git a/bitbake/bin/bitbake-diffsigs b/bitbake/bin/bitbake-diffsigs
index 67c60dbb00..3b6ef8811c 100755
--- a/bitbake/bin/bitbake-diffsigs
+++ b/bitbake/bin/bitbake-diffsigs
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # bitbake-diffsigs
 # BitBake task signature data comparison utility
@@ -24,6 +24,7 @@ import warnings
 import fnmatch
 import optparse
 import logging
+import pickle
 
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 
@@ -121,7 +122,6 @@ else:
     if len(args) == 1:
         parser.print_help()
     else:
-        import cPickle
         try:
             if len(args) == 2:
                 output = bb.siggen.dump_sigfile(sys.argv[1])
diff --git a/bitbake/bin/bitbake-dumpsig b/bitbake/bin/bitbake-dumpsig
index ffaed1f457..58ba1cad04 100755
--- a/bitbake/bin/bitbake-dumpsig
+++ b/bitbake/bin/bitbake-dumpsig
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # bitbake-dumpsig
 # BitBake task signature dump utility
@@ -23,6 +23,7 @@ import sys
 import warnings
 import optparse
 import logging
+import pickle
 
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 
@@ -51,7 +52,6 @@ options, args = parser.parse_args(sys.argv)
 if len(args) == 1:
     parser.print_help()
 else:
-    import cPickle
     try:
         output = bb.siggen.dump_sigfile(args[1])
     except IOError as e:
diff --git a/bitbake/bin/bitbake-layers b/bitbake/bin/bitbake-layers
index d8ffa9592a..0c973dfd2f 100755
--- a/bitbake/bin/bitbake-layers
+++ b/bitbake/bin/bitbake-layers
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # This script has subcommands which operate against your bitbake layers, either
 # displaying useful information, or acting against them.
@@ -48,7 +48,6 @@ def logger_create(name, output=sys.stderr):
     logger.setLevel(logging.INFO)
     return logger
 
-
 def logger_setup_color(logger, color='auto'):
     from bb.msg import BBLogFormatter
     console = logging.StreamHandler(sys.stdout)
@@ -61,7 +60,6 @@ def logger_setup_color(logger, color='auto'):
 
 logger = logger_create('bitbake-layers', sys.stdout)
 
-
 def main():
     parser = argparse.ArgumentParser(
         description="BitBake layers utility",
@@ -78,6 +76,7 @@ def main():
     parser.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS,
                         help='show this help message and exit')
     subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>')
+    subparsers.required = True
 
     if global_args.debug:
         logger.setLevel(logging.DEBUG)
diff --git a/bitbake/bin/bitbake-prserv b/bitbake/bin/bitbake-prserv
index 03821446b7..f38d2dd882 100755
--- a/bitbake/bin/bitbake-prserv
+++ b/bitbake/bin/bitbake-prserv
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import os
 import sys,logging
 import optparse
diff --git a/bitbake/bin/bitbake-selftest b/bitbake/bin/bitbake-selftest
index 462eb1b2b4..1e6f35ef0c 100755
--- a/bitbake/bin/bitbake-selftest
+++ b/bitbake/bin/bitbake-selftest
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (C) 2012 Richard Purdie
 #
diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker
index 6a6b26b64a..5d062a23e9 100755
--- a/bitbake/bin/bitbake-worker
+++ b/bitbake/bin/bitbake-worker
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
 import sys
@@ -10,8 +10,12 @@ import bb
 import select
 import errno
 import signal
+import pickle
 from multiprocessing import Lock
 
+if sys.getfilesystemencoding() != "utf-8":
+    sys.exit("Please use a locale setting which supports utf-8.\nPython can't change the filesystem locale after loading so we need a utf-8 when python starts or things won't work.")
+
 # Users shouldn't be running this code directly
 if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"):
     print("bitbake-worker is meant for internal execution by bitbake itself, please don't use it standalone.")
@@ -30,19 +34,16 @@ if sys.argv[1].startswith("decafbadbad"):
 # updates to log files for use with tail
 try:
     if sys.stdout.name == '<stdout>':
-        sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+        import fcntl
+        fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL)
+        fl |= os.O_SYNC 
+        fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl)
+        #sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
 except:
     pass
 
 logger = logging.getLogger("BitBake")
 
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
-    bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
-
-
 worker_pipe = sys.stdout.fileno()
 bb.utils.nonblockingfd(worker_pipe)
 # Need to guard against multiprocessing being used in child processes
@@ -62,10 +63,10 @@ if 0:
     consolelog.setFormatter(conlogformat)
     logger.addHandler(consolelog)
 
-worker_queue = ""
+worker_queue = b""
 
 def worker_fire(event, d):
-    data = "<event>" + pickle.dumps(event) + "</event>"
+    data = b"<event>" + pickle.dumps(event) + b"</event>"
     worker_fire_prepickled(data)
 
 def worker_fire_prepickled(event):
@@ -91,7 +92,7 @@ def worker_child_fire(event, d):
     global worker_pipe
     global worker_pipe_lock
 
-    data = "<event>" + pickle.dumps(event) + "</event>"
+    data = b"<event>" + pickle.dumps(event) + b"</event>"
     try:
         worker_pipe_lock.acquire()
         worker_pipe.write(data)
@@ -251,7 +252,7 @@ def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, taskdepdat
                 bb.utils.process_profilelog(profname)
                 os._exit(ret)
     else:
-        for key, value in envbackup.iteritems():
+        for key, value in iter(envbackup.items()):
             if value is None:
                 del os.environ[key]
             else:
@@ -268,22 +269,22 @@ class runQueueWorkerPipe():
         if pipeout:
             pipeout.close()
         bb.utils.nonblockingfd(self.input)
-        self.queue = ""
+        self.queue = b""
 
     def read(self):
         start = len(self.queue)
         try:
-            self.queue = self.queue + self.input.read(102400)
+            self.queue = self.queue + (self.input.read(102400) or b"")
         except (OSError, IOError) as e:
             if e.errno != errno.EAGAIN:
                 raise
 
         end = len(self.queue)
-        index = self.queue.find("</event>")
+        index = self.queue.find(b"</event>")
         while index != -1:
             worker_fire_prepickled(self.queue[:index+8])
             self.queue = self.queue[index+8:]
-            index = self.queue.find("</event>")
+            index = self.queue.find(b"</event>")
         return (end > start)
 
     def close(self):
@@ -299,7 +300,7 @@ class BitbakeWorker(object):
     def __init__(self, din):
         self.input = din
         bb.utils.nonblockingfd(self.input)
-        self.queue = ""
+        self.queue = b""
         self.cookercfg = None
         self.databuilder = None
         self.data = None
@@ -336,12 +337,12 @@ class BitbakeWorker(object):
                 except (OSError, IOError):
                     pass
             if len(self.queue):
-                self.handle_item("cookerconfig", self.handle_cookercfg)
-                self.handle_item("workerdata", self.handle_workerdata)
-                self.handle_item("runtask", self.handle_runtask)
-                self.handle_item("finishnow", self.handle_finishnow)
-                self.handle_item("ping", self.handle_ping)
-                self.handle_item("quit", self.handle_quit)
+                self.handle_item(b"cookerconfig", self.handle_cookercfg)
+                self.handle_item(b"workerdata", self.handle_workerdata)
+                self.handle_item(b"runtask", self.handle_runtask)
+                self.handle_item(b"finishnow", self.handle_finishnow)
+                self.handle_item(b"ping", self.handle_ping)
+                self.handle_item(b"quit", self.handle_quit)
 
             for pipe in self.build_pipes:
                 self.build_pipes[pipe].read()
@@ -351,12 +352,12 @@ class BitbakeWorker(object):
 
 
     def handle_item(self, item, func):
-        if self.queue.startswith("<" + item + ">"):
-            index = self.queue.find("</" + item + ">")
+        if self.queue.startswith(b"<" + item + b">"):
+            index = self.queue.find(b"</" + item + b">")
             while index != -1:
                 func(self.queue[(len(item) + 2):index])
                 self.queue = self.queue[(index + len(item) + 3):]
-                index = self.queue.find("</" + item + ">")
+                index = self.queue.find(b"</" + item + b">")
 
     def handle_cookercfg(self, data):
         self.cookercfg = pickle.loads(data)
@@ -420,12 +421,12 @@ class BitbakeWorker(object):
         self.build_pipes[pid].close()
         del self.build_pipes[pid]
 
-        worker_fire_prepickled("<exitcode>" + pickle.dumps((task, status)) + "</exitcode>")
+        worker_fire_prepickled(b"<exitcode>" + pickle.dumps((task, status)) + b"</exitcode>")
 
     def handle_finishnow(self, _):
         if self.build_pids:
             logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids))
-            for k, v in self.build_pids.iteritems():
+            for k, v in iter(self.build_pids.items()):
                 try:
                     os.kill(-k, signal.SIGTERM)
                     os.waitpid(-1, 0)
@@ -435,6 +436,7 @@ class BitbakeWorker(object):
             self.build_pipes[pipe].read()
 
 try:
+    sys.stdin = sys.stdin.detach()
     worker = BitbakeWorker(sys.stdin)
     if not profiling:
         worker.serve()
diff --git a/bitbake/bin/bitdoc b/bitbake/bin/bitdoc
index defb3dd37a..2744678824 100755
--- a/bitbake/bin/bitdoc
+++ b/bitbake/bin/bitdoc
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # ex:ts=4:sw=4:sts=4:et
 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
diff --git a/bitbake/bin/image-writer b/bitbake/bin/image-writer
index e30ab45e31..7ab1d8c979 100755
--- a/bitbake/bin/image-writer
+++ b/bitbake/bin/image-writer
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright (c) 2012 Wind River Systems, Inc.
 #
@@ -24,9 +24,13 @@ try:
 except RuntimeError as exc:
     sys.exit(str(exc))
 
+from gi import pygtkcompat
+
+pygtkcompat.enable()
+pygtkcompat.enable_gtk(version='3.0')
+
 import gtk
 import optparse
-import pygtk
 
 from bb.ui.crumbs.hobwidget import HobAltButton, HobButton
 from bb.ui.crumbs.hig.crumbsmessagedialog import CrumbsMessageDialog
diff --git a/bitbake/bin/toaster-eventreplay b/bitbake/bin/toaster-eventreplay
index 615a7aed15..bdddb908b5 100755
--- a/bitbake/bin/toaster-eventreplay
+++ b/bitbake/bin/toaster-eventreplay
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # ex:ts=4:sw=4:sts=4:et
 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #