From 89f16624842a91e063dc9e6b98787d44c55e4900 Mon Sep 17 00:00:00 2001 From: Julien Stephan Date: Wed, 22 Nov 2023 12:08:16 +0100 Subject: devtool: add support for git submodules Adding the support of submodules required a lot of changes on the internal data structures: * initial_rev/startcommit used as a starting point for looking at new / updated commits was replaced by a dictionary where the keys are the submodule name ("." for main repo) and the values are the initial_rev/startcommit * the extractPatches function now extracts patch for the main repo and for all submodules and stores them in a hierarchical way describing the submodule path * store initial_rev/commit also for all submodules inside the recipe bbappend file * _export_patches now returns dictionaries that contains the 'patchdir' parameter (if any). This parameter is used to add the correct 'patchdir=' parameter on the recipe Also, recipe can extract a secondary git tree inside the workdir. By default, at the end of the do_patch function, there is a hook in devtool that commits everything that was modified to have a clean repository. It uses the command: "git add .; git commit ..." The issue here is that, it adds the secondary git tree as a submodule but in a wrong way. Doing "git add " declares a submodule but do not adds a url associated to it, and all following "git submodule foreach" commands will fail. So detect that a git tree was extracted inside S and correctly add it using "git submodule add ", so that it will be considered as a regular git submodule (From OE-Core rev: 900129cbdf25297a42ab5dbd02d1adbea405c935) Signed-off-by: Julien Stephan Signed-off-by: Alexandre Belloni Signed-off-by: Richard Purdie --- meta/lib/oe/patch.py | 64 ++++++++++++++++++++++++---------------------- meta/lib/oe/recipeutils.py | 27 +++++++++++-------- 2 files changed, 50 insertions(+), 41 deletions(-) (limited to 'meta/lib/oe') diff --git a/meta/lib/oe/patch.py b/meta/lib/oe/patch.py index 9b480b2b28..e4bb5a7839 100644 --- a/meta/lib/oe/patch.py +++ b/meta/lib/oe/patch.py @@ -461,41 +461,43 @@ class GitApplyTree(PatchTree): return (tmpfile, cmd) @staticmethod - def extractPatches(tree, startcommit, outdir, paths=None): + def extractPatches(tree, startcommits, outdir, paths=None): import tempfile import shutil tempdir = tempfile.mkdtemp(prefix='oepatch') try: - shellcmd = ["git", "format-patch", "--no-signature", "--no-numbered", startcommit, "-o", tempdir] - if paths: - shellcmd.append('--') - shellcmd.extend(paths) - out = runcmd(["sh", "-c", " ".join(shellcmd)], tree) - if out: - for srcfile in out.split(): - for encoding in ['utf-8', 'latin-1']: - patchlines = [] - outfile = None - try: - with open(srcfile, 'r', encoding=encoding) as f: - for line in f: - if line.startswith(GitApplyTree.patch_line_prefix): - outfile = line.split()[-1].strip() - continue - if line.startswith(GitApplyTree.ignore_commit_prefix): - continue - patchlines.append(line) - except UnicodeDecodeError: - continue - break - else: - raise PatchError('Unable to find a character encoding to decode %s' % srcfile) - - if not outfile: - outfile = os.path.basename(srcfile) - with open(os.path.join(outdir, outfile), 'w') as of: - for line in patchlines: - of.write(line) + for name, rev in startcommits.items(): + shellcmd = ["git", "format-patch", "--no-signature", "--no-numbered", rev, "-o", tempdir] + if paths: + shellcmd.append('--') + shellcmd.extend(paths) + out = runcmd(["sh", "-c", " ".join(shellcmd)], os.path.join(tree, name)) + if out: + for srcfile in out.split(): + for encoding in ['utf-8', 'latin-1']: + patchlines = [] + outfile = None + try: + with open(srcfile, 'r', encoding=encoding) as f: + for line in f: + if line.startswith(GitApplyTree.patch_line_prefix): + outfile = line.split()[-1].strip() + continue + if line.startswith(GitApplyTree.ignore_commit_prefix): + continue + patchlines.append(line) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to find a character encoding to decode %s' % srcfile) + + if not outfile: + outfile = os.path.basename(srcfile) + bb.utils.mkdirhier(os.path.join(outdir, name)) + with open(os.path.join(outdir, name, outfile), 'w') as of: + for line in patchlines: + of.write(line) finally: shutil.rmtree(tempdir) diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py index 695e981623..25b159bc1b 100644 --- a/meta/lib/oe/recipeutils.py +++ b/meta/lib/oe/recipeutils.py @@ -672,11 +672,11 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, destlayerdir: base directory of the layer to place the bbappend in (subdirectory path from there will be determined automatically) srcfiles: dict of source files to add to SRC_URI, where the value - is the full path to the file to be added, and the value is the - original filename as it would appear in SRC_URI or None if it - isn't already present. You may pass None for this parameter if - you simply want to specify your own content via the extralines - parameter. + is the full path to the file to be added, and the value is a + dict with 'path' key containing the original filename as it + would appear in SRC_URI or None if it isn't already present. + You may pass None for this parameter if you simply want to specify + your own content via the extralines parameter. install: dict mapping entries in srcfiles to a tuple of two elements: install path (*without* ${D} prefix) and permission value (as a string, e.g. '0644'). @@ -763,10 +763,9 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, copyfiles = {} if srcfiles: instfunclines = [] - for i, (newfile, origsrcfile) in enumerate(srcfiles.items()): - srcfile = origsrcfile + for i, (newfile, param) in enumerate(srcfiles.items()): srcurientry = None - if not srcfile: + if not 'path' in param or not param['path']: srcfile = os.path.basename(newfile) srcurientry = 'file://%s' % srcfile if params and params[i]: @@ -778,7 +777,10 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, appendline('SRC_URI:append%s' % appendoverride, '=', ' ' + srcurientry) else: appendline('SRC_URI', '+=', srcurientry) - copyfiles[newfile] = srcfile + param['path'] = srcfile + else: + srcfile = param['path'] + copyfiles[newfile] = param if install: institem = install.pop(newfile, None) if institem: @@ -901,7 +903,12 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, outdir = redirect_output else: outdir = appenddir - for newfile, srcfile in copyfiles.items(): + for newfile, param in copyfiles.items(): + srcfile = param['path'] + patchdir = param.get('patchdir', ".") + + if patchdir != ".": + newfile = os.path.join(os.path.split(newfile)[0], patchdir, os.path.split(newfile)[1]) filedest = os.path.join(outdir, destsubdir, os.path.basename(srcfile)) if os.path.abspath(newfile) != os.path.abspath(filedest): if newfile.startswith(tempfile.gettempdir()): -- cgit v1.2.3-54-g00ecf