1 files changed, 157 insertions, 53 deletions

diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index 407d168894..54bc3d7666 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -24,9 +24,9 @@ from collections import OrderedDict, defaultdict
 from bb.utils import vercmp_string
 
 # Help us to find places to insert values
-recipe_progression = ['SUMMARY', 'DESCRIPTION', 'AUTHOR', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()', 'BBCLASSEXTEND']
+recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()', 'BBCLASSEXTEND']
 # Variables that sometimes are a bit long but shouldn't be wrapped
-nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', r'SRC_URI\[(.+\.)?md5sum\]', r'SRC_URI\[(.+\.)?sha256sum\]']
+nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', r'SRC_URI\[(.+\.)?md5sum\]', r'SRC_URI\[(.+\.)?sha[0-9]+sum\]']
 list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM']
 meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
 
@@ -47,7 +47,7 @@ def simplify_history(history, d):
                 continue
             has_set = True
         elif event['op'] in ('append', 'prepend', 'postdot', 'predot'):
-            # Reminder: "append" and "prepend" mean += and =+ respectively, NOT _append / _prepend
+            # Reminder: "append" and "prepend" mean += and =+ respectively, NOT :append / :prepend
             if has_set:
                 continue
         ret_history.insert(0, event)
@@ -342,7 +342,7 @@ def patch_recipe(d, fn, varvalues, patch=False, relpath='', redirect_output=None
     def override_applicable(hevent):
         op = hevent['op']
         if '[' in op:
-            opoverrides = op.split('[')[1].split(']')[0].split('_')
+            opoverrides = op.split('[')[1].split(']')[0].split(':')
             for opoverride in opoverrides:
                 if not opoverride in overrides:
                     return False
@@ -368,13 +368,13 @@ def patch_recipe(d, fn, varvalues, patch=False, relpath='', redirect_output=None
                                 recipe_set = True
                     if not recipe_set:
                         for event in history:
-                            if event['op'].startswith('_remove'):
+                            if event['op'].startswith(':remove'):
                                 continue
                             if not override_applicable(event):
                                 continue
                             newvalue = value.replace(event['detail'], '')
-                            if newvalue == value and os.path.abspath(event['file']) == fn and event['op'].startswith('_'):
-                                op = event['op'].replace('[', '_').replace(']', '')
+                            if newvalue == value and os.path.abspath(event['file']) == fn and event['op'].startswith(':'):
+                                op = event['op'].replace('[', ':').replace(']', '')
                                 extravals[var + op] = None
                             value = newvalue
                             vals[var] = ('+=', value)
@@ -414,15 +414,13 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True, all_variants=F
 
     fetch_urls(d)
     if all_variants:
-        # Get files for other variants e.g. in the case of a SRC_URI_append
+        # Get files for other variants e.g. in the case of a SRC_URI:append
         localdata = bb.data.createCopy(d)
         variants = (localdata.getVar('BBCLASSEXTEND') or '').split()
         if variants:
             # Ensure we handle class-target if we're dealing with one of the variants
             variants.append('target')
             for variant in variants:
-                if variant.startswith("devupstream"):
-                    localdata.setVar('SRCPV', 'git')
                 localdata.setVar('CLASSOVERRIDE', 'class-%s' % variant)
                 fetch_urls(localdata)
 
@@ -666,19 +664,23 @@ def get_bbappend_path(d, destlayerdir, wildcardver=False):
     return (appendpath, pathok)
 
 
-def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None):
+def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None, params=None, update_original_recipe=False):
     """
     Writes a bbappend file for a recipe
     Parameters:
         rd: data dictionary for the recipe
         destlayerdir: base directory of the layer to place the bbappend in
             (subdirectory path from there will be determined automatically)
-        srcfiles: dict of source files to add to SRC_URI, where the value
-            is the full path to the file to be added, and the value is the
-            original filename as it would appear in SRC_URI or None if it
-            isn't already present. You may pass None for this parameter if
-            you simply want to specify your own content via the extralines
-            parameter.
+        srcfiles: dict of source files to add to SRC_URI, where the key
+            is the full path to the file to be added, and the value is a
+            dict with following optional keys:
+                path: the original filename as it would appear in SRC_URI
+                    or None if it isn't already present.
+                patchdir: the patchdir parameter
+                newname: the name to give to the new added file. None to use
+                    the default value: basename(path)
+            You may pass None for this parameter if you simply want to specify
+            your own content via the extralines parameter.
         install: dict mapping entries in srcfiles to a tuple of two elements:
             install path (*without* ${D} prefix) and permission value (as a
             string, e.g. '0644').
@@ -696,18 +698,32 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
         redirect_output:
             If specified, redirects writing the output file to the
             specified directory (for dry-run purposes)
+        params:
+            Parameters to use when adding entries to SRC_URI. If specified,
+            should be a list of dicts with the same length as srcfiles.
+        update_original_recipe:
+            Force to update the original recipe instead of creating/updating
+            a bbapend. destlayerdir must contain the original recipe
     """
 
     if not removevalues:
         removevalues = {}
 
-    # Determine how the bbappend should be named
-    appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver)
-    if not appendpath:
-        bb.error('Unable to determine layer directory containing %s' % recipefile)
-        return (None, None)
-    if not pathok:
-        bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath)))
+    recipefile = rd.getVar('FILE')
+    if update_original_recipe:
+        if destlayerdir not in recipefile:
+            bb.error("destlayerdir %s doesn't contain the original recipe (%s), cannot update it" % (destlayerdir, recipefile))
+            return (None, None)
+
+        appendpath = recipefile
+    else:
+        # Determine how the bbappend should be named
+        appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver)
+        if not appendpath:
+            bb.error('Unable to determine layer directory containing %s' % recipefile)
+            return (None, None)
+        if not pathok:
+            bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath)))
 
     appenddir = os.path.dirname(appendpath)
     if not redirect_output:
@@ -752,30 +768,48 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
             bbappendlines.append((varname, op, value))
 
     destsubdir = rd.getVar('PN')
-    if srcfiles:
-        bbappendlines.append(('FILESEXTRAPATHS_prepend', ':=', '${THISDIR}/${PN}:'))
+    if not update_original_recipe and srcfiles:
+        bbappendlines.append(('FILESEXTRAPATHS:prepend', ':=', '${THISDIR}/${PN}:'))
 
     appendoverride = ''
     if machine:
         bbappendlines.append(('PACKAGE_ARCH', '=', '${MACHINE_ARCH}'))
-        appendoverride = '_%s' % machine
+        appendoverride = ':%s' % machine
     copyfiles = {}
     if srcfiles:
         instfunclines = []
-        for newfile, origsrcfile in srcfiles.items():
-            srcfile = origsrcfile
+        for i, (newfile, param) in enumerate(srcfiles.items()):
             srcurientry = None
-            if not srcfile:
-                srcfile = os.path.basename(newfile)
+            if not 'path' in param or not param['path']:
+                if 'newname' in param and param['newname']:
+                    srcfile = param['newname']
+                else:
+                    srcfile = os.path.basename(newfile)
                 srcurientry = 'file://%s' % srcfile
+                oldentry = None
+                for uri in rd.getVar('SRC_URI').split():
+                    if srcurientry in uri:
+                        oldentry = uri
+                if params and params[i]:
+                    srcurientry = '%s;%s' % (srcurientry, ';'.join('%s=%s' % (k,v) for k,v in params[i].items()))
                 # Double-check it's not there already
                 # FIXME do we care if the entry is added by another bbappend that might go away?
                 if not srcurientry in rd.getVar('SRC_URI').split():
                     if machine:
-                        appendline('SRC_URI_append%s' % appendoverride, '=', ' ' + srcurientry)
+                        if oldentry:
+                            appendline('SRC_URI:remove%s' % appendoverride, '=', ' ' + oldentry)
+                        appendline('SRC_URI:append%s' % appendoverride, '=', ' ' + srcurientry)
                     else:
+                        if oldentry:
+                            if update_original_recipe:
+                                removevalues['SRC_URI'] = oldentry
+                            else:
+                                appendline('SRC_URI:remove', '=', oldentry)
                         appendline('SRC_URI', '+=', srcurientry)
-            copyfiles[newfile] = srcfile
+                param['path'] = srcfile
+            else:
+                srcfile = param['path']
+            copyfiles[newfile] = param
             if install:
                 institem = install.pop(newfile, None)
                 if institem:
@@ -784,9 +818,9 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
                     instdirline = 'install -d ${D}%s' % os.path.dirname(instdestpath)
                     if not instdirline in instfunclines:
                         instfunclines.append(instdirline)
-                    instfunclines.append('install -m %s ${WORKDIR}/%s ${D}%s' % (perms, os.path.basename(srcfile), instdestpath))
+                    instfunclines.append('install -m %s ${UNPACKDIR}/%s ${D}%s' % (perms, os.path.basename(srcfile), instdestpath))
         if instfunclines:
-            bbappendlines.append(('do_install_append%s()' % appendoverride, '', instfunclines))
+            bbappendlines.append(('do_install:append%s()' % appendoverride, '', instfunclines))
 
     if redirect_output:
         bb.note('Writing append file %s (dry-run)' % appendpath)
@@ -795,6 +829,8 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
         # multiple times per operation when we're handling overrides)
         if os.path.exists(appendpath) and not os.path.exists(outfile):
             shutil.copy2(appendpath, outfile)
+    elif update_original_recipe:
+        outfile = recipefile
     else:
         bb.note('Writing append file %s' % appendpath)
         outfile = appendpath
@@ -804,15 +840,15 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
         extvars = {'destsubdir': destsubdir}
 
         def appendfile_varfunc(varname, origvalue, op, newlines):
-            if varname == 'FILESEXTRAPATHS_prepend':
+            if varname == 'FILESEXTRAPATHS:prepend':
                 if origvalue.startswith('${THISDIR}/'):
-                    popline('FILESEXTRAPATHS_prepend')
+                    popline('FILESEXTRAPATHS:prepend')
                     extvars['destsubdir'] = rd.expand(origvalue.split('${THISDIR}/', 1)[1].rstrip(':'))
             elif varname == 'PACKAGE_ARCH':
                 if machine:
                     popline('PACKAGE_ARCH')
                     return (machine, None, 4, False)
-            elif varname.startswith('do_install_append'):
+            elif varname.startswith('do_install:append'):
                 func = popline(varname)
                 if func:
                     instfunclines = [line.strip() for line in origvalue.strip('\n').splitlines()]
@@ -824,7 +860,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
                 splitval = split_var_value(origvalue, assignment=False)
                 changed = False
                 removevar = varname
-                if varname in ['SRC_URI', 'SRC_URI_append%s' % appendoverride]:
+                if varname in ['SRC_URI', 'SRC_URI:append%s' % appendoverride]:
                     removevar = 'SRC_URI'
                     line = popline(varname)
                     if line:
@@ -853,11 +889,11 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
                     newvalue = splitval
                     if len(newvalue) == 1:
                         # Ensure it's written out as one line
-                        if '_append' in varname:
+                        if ':append' in varname:
                             newvalue = ' ' + newvalue[0]
                         else:
                             newvalue = newvalue[0]
-                    if not newvalue and (op in ['+=', '.='] or '_append' in varname):
+                    if not newvalue and (op in ['+=', '.='] or ':append' in varname):
                         # There's no point appending nothing
                         newvalue = None
                     if varname.endswith('()'):
@@ -898,7 +934,12 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
             outdir = redirect_output
         else:
             outdir = appenddir
-        for newfile, srcfile in copyfiles.items():
+        for newfile, param in copyfiles.items():
+            srcfile = param['path']
+            patchdir = param.get('patchdir', ".")
+
+            if patchdir != ".":
+                newfile = os.path.join(os.path.split(newfile)[0], patchdir, os.path.split(newfile)[1])
             filedest = os.path.join(outdir, destsubdir, os.path.basename(srcfile))
             if os.path.abspath(newfile) != os.path.abspath(filedest):
                 if newfile.startswith(tempfile.gettempdir()):
@@ -942,10 +983,9 @@ def replace_dir_vars(path, d):
         path = path.replace(dirpath, '${%s}' % dirvars[dirpath])
     return path
 
-def get_recipe_pv_without_srcpv(pv, uri_type):
+def get_recipe_pv_with_pfx_sfx(pv, uri_type):
     """
-    Get PV without SRCPV common in SCM's for now only
-    support git.
+    Get PV separating prefix and suffix components.
 
     Returns tuple with pv, prefix and suffix.
     """
@@ -953,7 +993,7 @@ def get_recipe_pv_without_srcpv(pv, uri_type):
     sfx = ''
 
     if uri_type == 'git':
-        git_regex = re.compile(r"(?P<pfx>v?)(?P<ver>.*?)(?P<sfx>\+[^\+]*(git)?r?(AUTOINC\+))(?P<rev>.*)")
+        git_regex = re.compile(r"(?P<pfx>v?)(?P<ver>.*?)(?P<sfx>\+[^\+]*(git)?r?(AUTOINC\+)?)(?P<rev>.*)")
         m = git_regex.match(pv)
 
         if m:
@@ -1005,7 +1045,7 @@ def get_recipe_upstream_version(rd):
     src_uri = src_uris.split()[0]
     uri_type, _, _, _, _, _ =  decodeurl(src_uri)
 
-    (pv, pfx, sfx) = get_recipe_pv_without_srcpv(rd.getVar('PV'), uri_type)
+    (pv, pfx, sfx) = get_recipe_pv_with_pfx_sfx(rd.getVar('PV'), uri_type)
     ru['current_version'] = pv
 
     manual_upstream_version = rd.getVar("RECIPE_UPSTREAM_VERSION")
@@ -1029,10 +1069,16 @@ def get_recipe_upstream_version(rd):
     else:
         ud = bb.fetch2.FetchData(src_uri, rd)
         if rd.getVar("UPSTREAM_CHECK_COMMITS") == "1":
-            revision = ud.method.latest_revision(ud, rd, 'default')
-            upversion = pv
-            if revision != rd.getVar("SRCREV"):
-                upversion = upversion + "-new-commits-available" 
+            bb.fetch2.get_srcrev(rd)
+            upversion = None
+            revision = None
+            try:
+                revision = ud.method.latest_revision(ud, rd, ud.name)
+                upversion = pv
+                if revision != ud.revision:
+                    upversion = upversion + "-new-commits-available"
+            except bb.fetch2.FetchError as e:
+                bb.warn("Unable to obtain latest revision: {}".format(e))
         else:
             pupver = ud.method.latest_versionstring(ud, rd)
             (upversion, revision) = pupver
@@ -1071,7 +1117,7 @@ def _get_recipe_upgrade_status(data):
     maintainer = data.getVar('RECIPE_MAINTAINER')
     no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON')
 
-    return (pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason)
+    return {'pn':pn, 'status':status, 'cur_ver':cur_ver, 'next_ver':next_ver, 'maintainer':maintainer, 'revision':revision, 'no_upgrade_reason':no_upgrade_reason}
 
 def get_recipe_upgrade_status(recipes=None):
     pkgs_list = []
@@ -1113,6 +1159,7 @@ def get_recipe_upgrade_status(recipes=None):
         if not recipes:
             recipes = tinfoil.all_recipe_files(variants=False)
 
+        recipeincludes = {}
         for fn in recipes:
             try:
                 if fn.startswith("/"):
@@ -1137,8 +1184,65 @@ def get_recipe_upgrade_status(recipes=None):
 
             data_copy_list.append(data_copy)
 
+            recipeincludes[data.getVar('FILE')] = {'bbincluded':data.getVar('BBINCLUDED').split(),'pn':data.getVar('PN')}
+
     from concurrent.futures import ProcessPoolExecutor
     with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor:
         pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list)
 
-    return pkgs_list
+    return _group_recipes(pkgs_list, _get_common_include_recipes(recipeincludes))
+
+def get_common_include_recipes():
+    with bb.tinfoil.Tinfoil() as tinfoil:
+        tinfoil.prepare(config_only=False)
+
+        recipes = tinfoil.all_recipe_files(variants=False)
+
+        recipeincludes = {}
+        for fn in recipes:
+            data = tinfoil.parse_recipe_file(fn)
+            recipeincludes[fn] = {'bbincluded':data.getVar('BBINCLUDED').split(),'pn':data.getVar('PN')}
+        return _get_common_include_recipes(recipeincludes)
+
+def _get_common_include_recipes(recipeincludes_all):
+        recipeincludes = {}
+        for fn,data in recipeincludes_all.items():
+            bbincluded_filtered = [i for i in data['bbincluded'] if os.path.dirname(i) == os.path.dirname(fn) and i != fn]
+            if bbincluded_filtered:
+                recipeincludes[data['pn']] = bbincluded_filtered
+
+        recipeincludes_inverted = {}
+        for k,v in recipeincludes.items():
+            for i in v:
+                recipeincludes_inverted.setdefault(i,set()).add(k)
+
+        recipeincludes_inverted_filtered = {k:v for k,v in recipeincludes_inverted.items() if len(v) > 1}
+
+        recipes_with_shared_includes = list()
+        for v in recipeincludes_inverted_filtered.values():
+            recipeset = v
+            for v1 in recipeincludes_inverted_filtered.values():
+                if recipeset.intersection(v1):
+                    recipeset.update(v1)
+            if recipeset not in recipes_with_shared_includes:
+                recipes_with_shared_includes.append(recipeset)
+
+        return recipes_with_shared_includes
+
+def _group_recipes(recipes, groups):
+    recipedict = {}
+    for r in recipes:
+        recipedict[r['pn']] = r
+
+    recipegroups = []
+    for g in groups:
+        recipeset = []
+        for r in g:
+            if r in recipedict.keys():
+                recipeset.append(recipedict[r])
+                del recipedict[r]
+        recipegroups.append(recipeset)
+
+    for r in recipedict.values():
+        recipegroups.append([r])
+    return recipegroups