From bc8971d122a02ed823acf0758da267dccc584f98 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Thu, 12 May 2016 08:30:35 +0100 Subject: bitbake: bitbake: Convert to python 3 megacommit This needs breaking up into smaller changes. (Bitbake rev: cf51f19aed208a75d38c14cd585d9b9f115e3ba3) Signed-off-by: Richard Purdie --- bitbake/lib/bb/fetch2/__init__.py | 38 ++++++++++++++----------------- bitbake/lib/bb/fetch2/local.py | 4 ++-- bitbake/lib/bb/fetch2/npm.py | 6 ++--- bitbake/lib/bb/fetch2/perforce.py | 2 +- bitbake/lib/bb/fetch2/sftp.py | 8 +++---- bitbake/lib/bb/fetch2/ssh.py | 6 ++--- bitbake/lib/bb/fetch2/wget.py | 48 +++++++++++++++++++-------------------- 7 files changed, 52 insertions(+), 60 deletions(-) (limited to 'bitbake/lib/bb/fetch2') diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index 14fe3c753a..af7162b153 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py @@ -28,27 +28,23 @@ BitBake build tools. import os, re import signal import logging -import urllib -import urlparse +import urllib.request, urllib.parse, urllib.error +if 'git' not in urllib.parse.uses_netloc: + urllib.parse.uses_netloc.append('git') +import operator import collections +import subprocess +import pickle import bb.persist_data, bb.utils import bb.checksum from bb import data import bb.process -import subprocess __version__ = "2" _checksum_cache = bb.checksum.FileChecksumCache() logger = logging.getLogger("BitBake.Fetcher") -try: - import cPickle as pickle -except ImportError: - import pickle - logger.info("Importing cPickle failed. " - "Falling back to a very slow implementation.") - class BBFetchException(Exception): """Class all fetch exceptions inherit from""" def __init__(self, message): @@ -230,14 +226,14 @@ class URI(object): # them are not quite RFC compliant. uri, param_str = (uri.split(";", 1) + [None])[:2] - urlp = urlparse.urlparse(uri) + urlp = urllib.parse.urlparse(uri) self.scheme = urlp.scheme reparse = 0 # Coerce urlparse to make URI scheme use netloc - if not self.scheme in urlparse.uses_netloc: - urlparse.uses_params.append(self.scheme) + if not self.scheme in urllib.parse.uses_netloc: + urllib.parse.uses_params.append(self.scheme) reparse = 1 # Make urlparse happy(/ier) by converting local resources @@ -248,7 +244,7 @@ class URI(object): reparse = 1 if reparse: - urlp = urlparse.urlparse(uri) + urlp = urllib.parse.urlparse(uri) # Identify if the URI is relative or not if urlp.scheme in self._relative_schemes and \ @@ -264,7 +260,7 @@ class URI(object): if urlp.password: self.userinfo += ':%s' % urlp.password - self.path = urllib.unquote(urlp.path) + self.path = urllib.parse.unquote(urlp.path) if param_str: self.params = self._param_str_split(param_str, ";") @@ -312,11 +308,11 @@ class URI(object): @property def path_quoted(self): - return urllib.quote(self.path) + return urllib.parse.quote(self.path) @path_quoted.setter def path_quoted(self, path): - self.path = urllib.unquote(path) + self.path = urllib.parse.unquote(path) @property def path(self): @@ -398,7 +394,7 @@ def decodeurl(url): s1, s2 = s.split('=') p[s1] = s2 - return type, host, urllib.unquote(path), user, pswd, p + return type, host, urllib.parse.unquote(path), user, pswd, p def encodeurl(decoded): """Encodes a URL from tokens (scheme, network location, path, @@ -422,7 +418,7 @@ def encodeurl(decoded): # Standardise path to ensure comparisons work while '//' in path: path = path.replace("//", "/") - url += "%s" % urllib.quote(path) + url += "%s" % urllib.parse.quote(path) if p: for parm in p: url += ";%s=%s" % (parm, p[parm]) @@ -847,7 +843,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None): raise FetchError(error_message) - return output + return output.decode("utf-8") def check_network_access(d, info = "", url = None): """ @@ -1735,7 +1731,7 @@ class FetchConnectionCache(object): del self.cache[cn] def close_connections(self): - for cn in self.cache.keys(): + for cn in list(self.cache.keys()): self.cache[cn].close() del self.cache[cn] diff --git a/bitbake/lib/bb/fetch2/local.py b/bitbake/lib/bb/fetch2/local.py index 303a52b638..51ca78d12b 100644 --- a/bitbake/lib/bb/fetch2/local.py +++ b/bitbake/lib/bb/fetch2/local.py @@ -26,7 +26,7 @@ BitBake build tools. # Based on functions from the base bb module, Copyright 2003 Holger Schurig import os -import urllib +import urllib.request, urllib.parse, urllib.error import bb import bb.utils from bb import data @@ -42,7 +42,7 @@ class Local(FetchMethod): def urldata_init(self, ud, d): # We don't set localfile as for this fetcher the file is already local! - ud.decodedurl = urllib.unquote(ud.url.split("://")[1].split(";")[0]) + ud.decodedurl = urllib.parse.unquote(ud.url.split("://")[1].split(";")[0]) ud.basename = os.path.basename(ud.decodedurl) ud.basepath = ud.decodedurl ud.needdonestamp = False diff --git a/bitbake/lib/bb/fetch2/npm.py b/bitbake/lib/bb/fetch2/npm.py index d9e46b2e8c..2fd43034ba 100644 --- a/bitbake/lib/bb/fetch2/npm.py +++ b/bitbake/lib/bb/fetch2/npm.py @@ -20,7 +20,7 @@ Usage in the recipe: import os import sys -import urllib +import urllib.request, urllib.parse, urllib.error import json import subprocess import signal @@ -196,9 +196,9 @@ class Npm(FetchMethod): optdepsfound[dep] = dependencies[dep] else: depsfound[dep] = dependencies[dep] - for dep, version in optdepsfound.iteritems(): + for dep, version in optdepsfound.items(): self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True) - for dep, version in depsfound.iteritems(): + for dep, version in depsfound.items(): self._getdependencies(dep, data[pkg]['deps'], version, d, ud) def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest): diff --git a/bitbake/lib/bb/fetch2/perforce.py b/bitbake/lib/bb/fetch2/perforce.py index 1aef246942..ce3cda2670 100644 --- a/bitbake/lib/bb/fetch2/perforce.py +++ b/bitbake/lib/bb/fetch2/perforce.py @@ -61,7 +61,7 @@ class Perforce(FetchMethod): keys.append(key) values.append(value) - parm = dict(zip(keys, values)) + parm = dict(list(zip(keys, values))) path = "//" + path.split(';')[0] host += ":%s" % (port) parm["cset"] = Perforce.getcset(d, path, host, user, pswd, parm) diff --git a/bitbake/lib/bb/fetch2/sftp.py b/bitbake/lib/bb/fetch2/sftp.py index cb2f753a8e..7989fccc75 100644 --- a/bitbake/lib/bb/fetch2/sftp.py +++ b/bitbake/lib/bb/fetch2/sftp.py @@ -61,8 +61,7 @@ SRC_URI = "sftp://user@host.example.com/dir/path.file.txt" import os import bb -import urllib -import commands +import urllib.request, urllib.parse, urllib.error from bb import data from bb.fetch2 import URI from bb.fetch2 import FetchMethod @@ -93,7 +92,7 @@ class SFTP(FetchMethod): else: ud.basename = os.path.basename(ud.path) - ud.localfile = data.expand(urllib.unquote(ud.basename), d) + ud.localfile = data.expand(urllib.parse.unquote(ud.basename), d) def download(self, ud, d): """Fetch urls""" @@ -121,8 +120,7 @@ class SFTP(FetchMethod): remote = '%s%s:%s' % (user, urlo.hostname, path) - cmd = '%s %s %s %s' % (basecmd, port, commands.mkarg(remote), - commands.mkarg(lpath)) + cmd = '%s %s %s %s' % (basecmd, port, remote, lpath) bb.fetch2.check_network_access(d, cmd, ud.url) runfetchcmd(cmd, d) diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py index 635578a711..56f9b7eb35 100644 --- a/bitbake/lib/bb/fetch2/ssh.py +++ b/bitbake/lib/bb/fetch2/ssh.py @@ -114,12 +114,10 @@ class SSH(FetchMethod): fr = host fr += ':%s' % path - - import commands cmd = 'scp -B -r %s %s %s/' % ( portarg, - commands.mkarg(fr), - commands.mkarg(dldir) + fr, + dldir ) bb.fetch2.check_network_access(d, cmd, urldata.url) diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py index 8bc9e93ca0..d688fd9d02 100644 --- a/bitbake/lib/bb/fetch2/wget.py +++ b/bitbake/lib/bb/fetch2/wget.py @@ -31,7 +31,7 @@ import subprocess import os import logging import bb -import urllib +import urllib.request, urllib.parse, urllib.error from bb import data from bb.fetch2 import FetchMethod from bb.fetch2 import FetchError @@ -62,9 +62,9 @@ class Wget(FetchMethod): else: ud.basename = os.path.basename(ud.path) - ud.localfile = data.expand(urllib.unquote(ud.basename), d) + ud.localfile = data.expand(urllib.parse.unquote(ud.basename), d) if not ud.localfile: - ud.localfile = data.expand(urllib.unquote(ud.host + ud.path).replace("/", "."), d) + ud.localfile = data.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."), d) self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 -nv --passive-ftp --no-check-certificate" @@ -105,11 +105,11 @@ class Wget(FetchMethod): return True def checkstatus(self, fetch, ud, d): - import urllib2, socket, httplib - from urllib import addinfourl + import urllib.request, urllib.error, urllib.parse, socket, http.client + from urllib.response import addinfourl from bb.fetch2 import FetchConnectionCache - class HTTPConnectionCache(httplib.HTTPConnection): + class HTTPConnectionCache(http.client.HTTPConnection): if fetch.connection_cache: def connect(self): """Connect to the host and port specified in __init__.""" @@ -125,7 +125,7 @@ class Wget(FetchMethod): if self._tunnel_host: self._tunnel() - class CacheHTTPHandler(urllib2.HTTPHandler): + class CacheHTTPHandler(urllib.request.HTTPHandler): def http_open(self, req): return self.do_open(HTTPConnectionCache, req) @@ -139,7 +139,7 @@ class Wget(FetchMethod): - geturl(): return the original request URL - code: HTTP status code """ - host = req.get_host() + host = req.host if not host: raise urlllib2.URLError('no host given') @@ -147,7 +147,7 @@ class Wget(FetchMethod): h.set_debuglevel(self._debuglevel) headers = dict(req.unredirected_hdrs) - headers.update(dict((k, v) for k, v in req.headers.items() + headers.update(dict((k, v) for k, v in list(req.headers.items()) if k not in headers)) # We want to make an HTTP/1.1 request, but the addinfourl @@ -164,7 +164,7 @@ class Wget(FetchMethod): headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0 headers = dict( - (name.title(), val) for name, val in headers.items()) + (name.title(), val) for name, val in list(headers.items())) if req._tunnel_host: tunnel_headers = {} @@ -177,12 +177,12 @@ class Wget(FetchMethod): h.set_tunnel(req._tunnel_host, headers=tunnel_headers) try: - h.request(req.get_method(), req.get_selector(), req.data, headers) - except socket.error, err: # XXX what error? + h.request(req.get_method(), req.selector, req.data, headers) + except socket.error as err: # XXX what error? # Don't close connection when cache is enabled. if fetch.connection_cache is None: h.close() - raise urllib2.URLError(err) + raise urllib.error.URLError(err) else: try: r = h.getresponse(buffering=True) @@ -222,7 +222,7 @@ class Wget(FetchMethod): return resp - class HTTPMethodFallback(urllib2.BaseHandler): + class HTTPMethodFallback(urllib.request.BaseHandler): """ Fallback to GET if HEAD is not allowed (405 HTTP error) """ @@ -230,11 +230,11 @@ class Wget(FetchMethod): fp.read() fp.close() - newheaders = dict((k,v) for k,v in req.headers.items() + newheaders = dict((k,v) for k,v in list(req.headers.items()) if k.lower() not in ("content-length", "content-type")) - return self.parent.open(urllib2.Request(req.get_full_url(), + return self.parent.open(urllib.request.Request(req.get_full_url(), headers=newheaders, - origin_req_host=req.get_origin_req_host(), + origin_req_host=req.origin_req_host, unverifiable=True)) """ @@ -249,35 +249,35 @@ class Wget(FetchMethod): """ http_error_406 = http_error_405 - class FixedHTTPRedirectHandler(urllib2.HTTPRedirectHandler): + class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler): """ urllib2.HTTPRedirectHandler resets the method to GET on redirect, when we want to follow redirects using the original method. """ def redirect_request(self, req, fp, code, msg, headers, newurl): - newreq = urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) + newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) newreq.get_method = lambda: req.get_method() return newreq exported_proxies = export_proxies(d) handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] if export_proxies: - handlers.append(urllib2.ProxyHandler()) + handlers.append(urllib.request.ProxyHandler()) handlers.append(CacheHTTPHandler()) # XXX: Since Python 2.7.9 ssl cert validation is enabled by default # see PEP-0476, this causes verification errors on some https servers # so disable by default. import ssl if hasattr(ssl, '_create_unverified_context'): - handlers.append(urllib2.HTTPSHandler(context=ssl._create_unverified_context())) - opener = urllib2.build_opener(*handlers) + handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context())) + opener = urllib.request.build_opener(*handlers) try: uri = ud.url.split(";")[0] - r = urllib2.Request(uri) + r = urllib.request.Request(uri) r.get_method = lambda: "HEAD" opener.open(r) - except urllib2.URLError as e: + except urllib.error.URLError as e: # debug for now to avoid spamming the logs in e.g. remote sstate searches logger.debug(2, "checkstatus() urlopen failed: %s" % e) return False -- cgit v1.2.3-54-g00ecf