summaryrefslogtreecommitdiffstats
path: root/scripts/lib
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/lib')
-rw-r--r--scripts/lib/checklayer/cases/common.py13
-rw-r--r--scripts/lib/recipetool/create.py58
-rw-r--r--scripts/lib/recipetool/create_buildsys.py3
-rw-r--r--scripts/lib/recipetool/create_buildsys_python.py8
-rw-r--r--scripts/lib/scriptutils.py7
-rw-r--r--scripts/lib/wic/pluginbase.py8
6 files changed, 70 insertions, 27 deletions
diff --git a/scripts/lib/checklayer/cases/common.py b/scripts/lib/checklayer/cases/common.py
index b82304e..9f15e05 100644
--- a/scripts/lib/checklayer/cases/common.py
+++ b/scripts/lib/checklayer/cases/common.py
@@ -6,6 +6,7 @@
6import glob 6import glob
7import os 7import os
8import unittest 8import unittest
9import re
9from checklayer import get_signatures, LayerType, check_command, get_depgraph, compare_signatures 10from checklayer import get_signatures, LayerType, check_command, get_depgraph, compare_signatures
10from checklayer.case import OECheckLayerTestCase 11from checklayer.case import OECheckLayerTestCase
11 12
@@ -14,7 +15,7 @@ class CommonCheckLayer(OECheckLayerTestCase):
14 # The top-level README file may have a suffix (like README.rst or README.txt). 15 # The top-level README file may have a suffix (like README.rst or README.txt).
15 readme_files = glob.glob(os.path.join(self.tc.layer['path'], '[Rr][Ee][Aa][Dd][Mm][Ee]*')) 16 readme_files = glob.glob(os.path.join(self.tc.layer['path'], '[Rr][Ee][Aa][Dd][Mm][Ee]*'))
16 self.assertTrue(len(readme_files) > 0, 17 self.assertTrue(len(readme_files) > 0,
17 msg="Layer doesn't contains README file.") 18 msg="Layer doesn't contain a README file.")
18 19
19 # There might be more than one file matching the file pattern above 20 # There might be more than one file matching the file pattern above
20 # (for example, README.rst and README-COPYING.rst). The one with the shortest 21 # (for example, README.rst and README-COPYING.rst). The one with the shortest
@@ -26,6 +27,16 @@ class CommonCheckLayer(OECheckLayerTestCase):
26 self.assertTrue(data, 27 self.assertTrue(data,
27 msg="Layer contains a README file but it is empty.") 28 msg="Layer contains a README file but it is empty.")
28 29
30 # If a layer's README references another README, then the checks below are not valid
31 if re.search('README', data, re.IGNORECASE):
32 return
33
34 self.assertIn('maintainer', data.lower())
35 self.assertIn('patch', data.lower())
36 # Check that there is an email address in the README
37 email_regex = re.compile(r"[^@]+@[^@]+")
38 self.assertTrue(email_regex.match(data))
39
29 def test_parse(self): 40 def test_parse(self):
30 check_command('Layer %s failed to parse.' % self.tc.layer['name'], 41 check_command('Layer %s failed to parse.' % self.tc.layer['name'],
31 'bitbake -p') 42 'bitbake -p')
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index b6c4564..507a230 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -366,7 +366,7 @@ def supports_srcrev(uri):
366def reformat_git_uri(uri): 366def reformat_git_uri(uri):
367 '''Convert any http[s]://....git URI into git://...;protocol=http[s]''' 367 '''Convert any http[s]://....git URI into git://...;protocol=http[s]'''
368 checkuri = uri.split(';', 1)[0] 368 checkuri = uri.split(';', 1)[0]
369 if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://github.com/[^/]+/[^/]+/?$', checkuri): 369 if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://git(hub|lab).com/[^/]+/[^/]+/?$', checkuri):
370 # Appends scheme if the scheme is missing 370 # Appends scheme if the scheme is missing
371 if not '://' in uri: 371 if not '://' in uri:
372 uri = 'git://' + uri 372 uri = 'git://' + uri
@@ -389,9 +389,6 @@ def reformat_git_uri(uri):
389 parms.update({('protocol', 'ssh')}) 389 parms.update({('protocol', 'ssh')})
390 elif (scheme == "http" or scheme == 'https' or scheme == 'ssh') and not ('protocol' in parms): 390 elif (scheme == "http" or scheme == 'https' or scheme == 'ssh') and not ('protocol' in parms):
391 parms.update({('protocol', scheme)}) 391 parms.update({('protocol', scheme)})
392 # We assume 'master' branch if not set
393 if not 'branch' in parms:
394 parms.update({('branch', 'master')})
395 # Always append 'git://' 392 # Always append 'git://'
396 fUrl = bb.fetch2.encodeurl(('git', host, path, user, pswd, parms)) 393 fUrl = bb.fetch2.encodeurl(('git', host, path, user, pswd, parms))
397 return fUrl 394 return fUrl
@@ -481,6 +478,9 @@ def create_recipe(args):
481 storeTagName = params['tag'] 478 storeTagName = params['tag']
482 params['nobranch'] = '1' 479 params['nobranch'] = '1'
483 del params['tag'] 480 del params['tag']
481 # Assume 'master' branch if not set
482 if scheme in ['git', 'gitsm'] and 'branch' not in params and 'nobranch' not in params:
483 params['branch'] = 'master'
484 fetchuri = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params)) 484 fetchuri = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params))
485 485
486 tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR') 486 tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR')
@@ -530,10 +530,9 @@ def create_recipe(args):
530 # Remove HEAD reference point and drop remote prefix 530 # Remove HEAD reference point and drop remote prefix
531 get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')] 531 get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')]
532 if 'master' in get_branch: 532 if 'master' in get_branch:
533 # If it is master, we do not need to append 'branch=master' as this is default.
534 # Even with the case where get_branch has multiple objects, if 'master' is one 533 # Even with the case where get_branch has multiple objects, if 'master' is one
535 # of them, we should default take from 'master' 534 # of them, we should default take from 'master'
536 srcbranch = '' 535 srcbranch = 'master'
537 elif len(get_branch) == 1: 536 elif len(get_branch) == 1:
538 # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch' 537 # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch'
539 srcbranch = get_branch[0] 538 srcbranch = get_branch[0]
@@ -546,8 +545,8 @@ def create_recipe(args):
546 # Since we might have a value in srcbranch, we need to 545 # Since we might have a value in srcbranch, we need to
547 # recontruct the srcuri to include 'branch' in params. 546 # recontruct the srcuri to include 'branch' in params.
548 scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(srcuri) 547 scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(srcuri)
549 if srcbranch: 548 if scheme in ['git', 'gitsm']:
550 params['branch'] = srcbranch 549 params['branch'] = srcbranch or 'master'
551 550
552 if storeTagName and scheme in ['git', 'gitsm']: 551 if storeTagName and scheme in ['git', 'gitsm']:
553 # Check srcrev using tag and check validity of the tag 552 # Check srcrev using tag and check validity of the tag
@@ -606,7 +605,7 @@ def create_recipe(args):
606 splitline = line.split() 605 splitline = line.split()
607 if len(splitline) > 1: 606 if len(splitline) > 1:
608 if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]): 607 if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]):
609 srcuri = reformat_git_uri(splitline[1]) 608 srcuri = reformat_git_uri(splitline[1]) + ';branch=master'
610 srcsubdir = 'git' 609 srcsubdir = 'git'
611 break 610 break
612 611
@@ -920,6 +919,22 @@ def split_value(value):
920 else: 919 else:
921 return value 920 return value
922 921
922def fixup_license(value):
923 # Ensure licenses with OR starts and ends with brackets
924 if '|' in value:
925 return '(' + value + ')'
926 return value
927
928def tidy_licenses(value):
929 """Flat, split and sort licenses"""
930 from oe.license import flattened_licenses
931 def _choose(a, b):
932 str_a, str_b = sorted((" & ".join(a), " & ".join(b)), key=str.casefold)
933 return ["(%s | %s)" % (str_a, str_b)]
934 if not isinstance(value, str):
935 value = " & ".join(value)
936 return sorted(list(set(flattened_licenses(value, _choose))), key=str.casefold)
937
923def handle_license_vars(srctree, lines_before, handled, extravalues, d): 938def handle_license_vars(srctree, lines_before, handled, extravalues, d):
924 lichandled = [x for x in handled if x[0] == 'license'] 939 lichandled = [x for x in handled if x[0] == 'license']
925 if lichandled: 940 if lichandled:
@@ -933,10 +948,13 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
933 lines = [] 948 lines = []
934 if licvalues: 949 if licvalues:
935 for licvalue in licvalues: 950 for licvalue in licvalues:
936 if not licvalue[0] in licenses: 951 license = licvalue[0]
937 licenses.append(licvalue[0]) 952 lics = tidy_licenses(fixup_license(license))
953 lics = [lic for lic in lics if lic not in licenses]
954 if len(lics):
955 licenses.extend(lics)
938 lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) 956 lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
939 if licvalue[0] == 'Unknown': 957 if license == 'Unknown':
940 lic_unknown.append(licvalue[1]) 958 lic_unknown.append(licvalue[1])
941 if lic_unknown: 959 if lic_unknown:
942 lines.append('#') 960 lines.append('#')
@@ -945,9 +963,7 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
945 for licfile in lic_unknown: 963 for licfile in lic_unknown:
946 lines.append('# %s' % licfile) 964 lines.append('# %s' % licfile)
947 965
948 extra_license = split_value(extravalues.pop('LICENSE', [])) 966 extra_license = tidy_licenses(extravalues.pop('LICENSE', ''))
949 if '&' in extra_license:
950 extra_license.remove('&')
951 if extra_license: 967 if extra_license:
952 if licenses == ['Unknown']: 968 if licenses == ['Unknown']:
953 licenses = extra_license 969 licenses = extra_license
@@ -988,7 +1004,7 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
988 lines.append('# instead of &. If there is any doubt, check the accompanying documentation') 1004 lines.append('# instead of &. If there is any doubt, check the accompanying documentation')
989 lines.append('# to determine which situation is applicable.') 1005 lines.append('# to determine which situation is applicable.')
990 1006
991 lines.append('LICENSE = "%s"' % ' & '.join(licenses)) 1007 lines.append('LICENSE = "%s"' % ' & '.join(sorted(licenses, key=str.casefold)))
992 lines.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) 1008 lines.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum))
993 lines.append('') 1009 lines.append('')
994 1010
@@ -1202,7 +1218,7 @@ def guess_license(srctree, d):
1202 fullpath = os.path.join(root, fn) 1218 fullpath = os.path.join(root, fn)
1203 if not fullpath in licfiles: 1219 if not fullpath in licfiles:
1204 licfiles.append(fullpath) 1220 licfiles.append(fullpath)
1205 for licfile in licfiles: 1221 for licfile in sorted(licfiles):
1206 md5value = bb.utils.md5_file(licfile) 1222 md5value = bb.utils.md5_file(licfile)
1207 license = md5sums.get(md5value, None) 1223 license = md5sums.get(md5value, None)
1208 if not license: 1224 if not license:
@@ -1227,6 +1243,7 @@ def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=[], pn='
1227 """ 1243 """
1228 pkglicenses = {pn: []} 1244 pkglicenses = {pn: []}
1229 for license, licpath, _ in licvalues: 1245 for license, licpath, _ in licvalues:
1246 license = fixup_license(license)
1230 for pkgname, pkgpath in packages.items(): 1247 for pkgname, pkgpath in packages.items():
1231 if licpath.startswith(pkgpath + '/'): 1248 if licpath.startswith(pkgpath + '/'):
1232 if pkgname in pkglicenses: 1249 if pkgname in pkglicenses:
@@ -1239,11 +1256,14 @@ def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=[], pn='
1239 pkglicenses[pn].append(license) 1256 pkglicenses[pn].append(license)
1240 outlicenses = {} 1257 outlicenses = {}
1241 for pkgname in packages: 1258 for pkgname in packages:
1242 license = ' '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown' 1259 # Assume AND operator between license files
1260 license = ' & '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown'
1243 if license == 'Unknown' and pkgname in fallback_licenses: 1261 if license == 'Unknown' and pkgname in fallback_licenses:
1244 license = fallback_licenses[pkgname] 1262 license = fallback_licenses[pkgname]
1263 licenses = tidy_licenses(license)
1264 license = ' & '.join(licenses)
1245 outlines.append('LICENSE:%s = "%s"' % (pkgname, license)) 1265 outlines.append('LICENSE:%s = "%s"' % (pkgname, license))
1246 outlicenses[pkgname] = license.split() 1266 outlicenses[pkgname] = licenses
1247 return outlicenses 1267 return outlicenses
1248 1268
1249def read_pkgconfig_provides(d): 1269def read_pkgconfig_provides(d):
diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py
index 35a97c9..5015634 100644
--- a/scripts/lib/recipetool/create_buildsys.py
+++ b/scripts/lib/recipetool/create_buildsys.py
@@ -545,7 +545,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
545 deps.append('zlib') 545 deps.append('zlib')
546 elif keyword in ('AX_CHECK_OPENSSL', 'AX_LIB_CRYPTO'): 546 elif keyword in ('AX_CHECK_OPENSSL', 'AX_LIB_CRYPTO'):
547 deps.append('openssl') 547 deps.append('openssl')
548 elif keyword == 'AX_LIB_CURL': 548 elif keyword in ('AX_LIB_CURL', 'LIBCURL_CHECK_CONFIG'):
549 deps.append('curl') 549 deps.append('curl')
550 elif keyword == 'AX_LIB_BEECRYPT': 550 elif keyword == 'AX_LIB_BEECRYPT':
551 deps.append('beecrypt') 551 deps.append('beecrypt')
@@ -624,6 +624,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
624 'AX_CHECK_OPENSSL', 624 'AX_CHECK_OPENSSL',
625 'AX_LIB_CRYPTO', 625 'AX_LIB_CRYPTO',
626 'AX_LIB_CURL', 626 'AX_LIB_CURL',
627 'LIBCURL_CHECK_CONFIG',
627 'AX_LIB_BEECRYPT', 628 'AX_LIB_BEECRYPT',
628 'AX_LIB_EXPAT', 629 'AX_LIB_EXPAT',
629 'AX_LIB_GCRYPT', 630 'AX_LIB_GCRYPT',
diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py
index 0b6b042..1a15011 100644
--- a/scripts/lib/recipetool/create_buildsys_python.py
+++ b/scripts/lib/recipetool/create_buildsys_python.py
@@ -8,7 +8,7 @@
8import ast 8import ast
9import codecs 9import codecs
10import collections 10import collections
11import distutils.command.build_py 11import setuptools.command.build_py
12import email 12import email
13import imp 13import imp
14import glob 14import glob
@@ -459,9 +459,13 @@ class PythonRecipeHandler(RecipeHandler):
459 else: 459 else:
460 package_dir = {} 460 package_dir = {}
461 461
462 class PackageDir(distutils.command.build_py.build_py): 462 dist = setuptools.Distribution()
463
464 class PackageDir(setuptools.command.build_py.build_py):
463 def __init__(self, package_dir): 465 def __init__(self, package_dir):
464 self.package_dir = package_dir 466 self.package_dir = package_dir
467 self.dist = dist
468 super().__init__(self.dist)
465 469
466 pd = PackageDir(package_dir) 470 pd = PackageDir(package_dir)
467 to_scan = [] 471 to_scan = []
diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py
index 3164171..47a0819 100644
--- a/scripts/lib/scriptutils.py
+++ b/scripts/lib/scriptutils.py
@@ -18,7 +18,8 @@ import sys
18import tempfile 18import tempfile
19import threading 19import threading
20import importlib 20import importlib
21from importlib import machinery 21import importlib.machinery
22import importlib.util
22 23
23class KeepAliveStreamHandler(logging.StreamHandler): 24class KeepAliveStreamHandler(logging.StreamHandler):
24 def __init__(self, keepalive=True, **kwargs): 25 def __init__(self, keepalive=True, **kwargs):
@@ -82,7 +83,9 @@ def load_plugins(logger, plugins, pluginpath):
82 logger.debug('Loading plugin %s' % name) 83 logger.debug('Loading plugin %s' % name)
83 spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] ) 84 spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] )
84 if spec: 85 if spec:
85 return spec.loader.load_module() 86 mod = importlib.util.module_from_spec(spec)
87 spec.loader.exec_module(mod)
88 return mod
86 89
87 def plugin_name(filename): 90 def plugin_name(filename):
88 return os.path.splitext(os.path.basename(filename))[0] 91 return os.path.splitext(os.path.basename(filename))[0]
diff --git a/scripts/lib/wic/pluginbase.py b/scripts/lib/wic/pluginbase.py
index d9b4e57..b645683 100644
--- a/scripts/lib/wic/pluginbase.py
+++ b/scripts/lib/wic/pluginbase.py
@@ -9,9 +9,11 @@ __all__ = ['ImagerPlugin', 'SourcePlugin']
9 9
10import os 10import os
11import logging 11import logging
12import types
12 13
13from collections import defaultdict 14from collections import defaultdict
14from importlib.machinery import SourceFileLoader 15import importlib
16import importlib.util
15 17
16from wic import WicError 18from wic import WicError
17from wic.misc import get_bitbake_var 19from wic.misc import get_bitbake_var
@@ -54,7 +56,9 @@ class PluginMgr:
54 mname = fname[:-3] 56 mname = fname[:-3]
55 mpath = os.path.join(ppath, fname) 57 mpath = os.path.join(ppath, fname)
56 logger.debug("loading plugin module %s", mpath) 58 logger.debug("loading plugin module %s", mpath)
57 SourceFileLoader(mname, mpath).load_module() 59 spec = importlib.util.spec_from_file_location(mname, mpath)
60 module = importlib.util.module_from_spec(spec)
61 spec.loader.exec_module(module)
58 62
59 return PLUGINS.get(ptype) 63 return PLUGINS.get(ptype)
60 64