summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2024-05-01 22:36:50 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2024-05-21 12:08:04 +0100
commit6d77296d83f87ea7f6ca0363a947d3eac721459b (patch)
treebdd4f17e69cffdcc39e20e43a917938736482e45
parentd6ae8d2004f8ac3aab4571e0077d406b2d19b542 (diff)
downloadpoky-6d77296d83f87ea7f6ca0363a947d3eac721459b.tar.gz
devtool: Drop oe-local-files and simplify
The only real reason for oe-local-files was to support S = WORKDIR. With changes to drop support for that, it makes sense to simplify devtool and to try and make both the code and the processes/workflows simpler. This patch drops support for S = WORKDIR, removes oe-local-files and then updates the test cases to match this new situation. At the code level, we assume we can always now track code changes using git and that things committed into git are handled as patches (as before) but delta against HEAD is saved as specific file level changes to the recipe. One test is disabled as it is no longer approproate. It is being keped until we can make WORKDIR != UNPACKDIR at which point it should be revisited. (From OE-Core rev: ce8190c519052fed10b5233697b69a75868db45a) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta-selftest/recipes-test/devtool/devtool-test-localonly.bb3
-rw-r--r--meta/classes-recipe/kernel-yocto.bbclass2
-rw-r--r--meta/classes/devtool-source.bbclass74
-rw-r--r--meta/lib/oeqa/selftest/cases/devtool.py38
-rw-r--r--scripts/lib/devtool/standard.py194
5 files changed, 89 insertions, 222 deletions
diff --git a/meta-selftest/recipes-test/devtool/devtool-test-localonly.bb b/meta-selftest/recipes-test/devtool/devtool-test-localonly.bb
index e767619879..446c51f09b 100644
--- a/meta-selftest/recipes-test/devtool/devtool-test-localonly.bb
+++ b/meta-selftest/recipes-test/devtool/devtool-test-localonly.bb
@@ -6,5 +6,8 @@ SRC_URI = "file://file1 \
6 6
7SRC_URI:append:class-native = " file://file3" 7SRC_URI:append:class-native = " file://file3"
8 8
9S = "${WORKDIR}/sources"
10UNPACKDIR = "${S}"
11
9EXCLUDE_FROM_WORLD = "1" 12EXCLUDE_FROM_WORLD = "1"
10BBCLASSEXTEND = "native" 13BBCLASSEXTEND = "native"
diff --git a/meta/classes-recipe/kernel-yocto.bbclass b/meta/classes-recipe/kernel-yocto.bbclass
index c4ed3f1ca2..e8ff7311c3 100644
--- a/meta/classes-recipe/kernel-yocto.bbclass
+++ b/meta/classes-recipe/kernel-yocto.bbclass
@@ -234,8 +234,6 @@ do_kernel_metadata() {
234 for f in ${feat_dirs}; do 234 for f in ${feat_dirs}; do
235 if [ -d "${UNPACKDIR}/$f/kernel-meta" ]; then 235 if [ -d "${UNPACKDIR}/$f/kernel-meta" ]; then
236 includes="$includes -I${UNPACKDIR}/$f/kernel-meta" 236 includes="$includes -I${UNPACKDIR}/$f/kernel-meta"
237 elif [ -d "${UNPACKDIR}/../oe-local-files/$f" ]; then
238 includes="$includes -I${UNPACKDIR}/../oe-local-files/$f"
239 elif [ -d "${UNPACKDIR}/$f" ]; then 237 elif [ -d "${UNPACKDIR}/$f" ]; then
240 includes="$includes -I${UNPACKDIR}/$f" 238 includes="$includes -I${UNPACKDIR}/$f"
241 fi 239 fi
diff --git a/meta/classes/devtool-source.bbclass b/meta/classes/devtool-source.bbclass
index 4158c20c7e..3e24800dcb 100644
--- a/meta/classes/devtool-source.bbclass
+++ b/meta/classes/devtool-source.bbclass
@@ -26,8 +26,6 @@
26 26
27 27
28DEVTOOL_TEMPDIR ?= "" 28DEVTOOL_TEMPDIR ?= ""
29DEVTOOL_PATCH_SRCDIR = "${DEVTOOL_TEMPDIR}/patchworkdir"
30
31 29
32python() { 30python() {
33 tempdir = d.getVar('DEVTOOL_TEMPDIR') 31 tempdir = d.getVar('DEVTOOL_TEMPDIR')
@@ -60,7 +58,6 @@ python() {
60 else: 58 else:
61 unpacktask = 'do_unpack' 59 unpacktask = 'do_unpack'
62 d.appendVarFlag(unpacktask, 'postfuncs', ' devtool_post_unpack') 60 d.appendVarFlag(unpacktask, 'postfuncs', ' devtool_post_unpack')
63 d.prependVarFlag('do_patch', 'prefuncs', ' devtool_pre_patch')
64 d.appendVarFlag('do_patch', 'postfuncs', ' devtool_post_patch') 61 d.appendVarFlag('do_patch', 'postfuncs', ' devtool_post_patch')
65 62
66 # NOTE: in order for the patch stuff to be fully functional, 63 # NOTE: in order for the patch stuff to be fully functional,
@@ -79,67 +76,23 @@ python devtool_post_unpack() {
79 76
80 tempdir = d.getVar('DEVTOOL_TEMPDIR') 77 tempdir = d.getVar('DEVTOOL_TEMPDIR')
81 workdir = d.getVar('WORKDIR') 78 workdir = d.getVar('WORKDIR')
79 unpackdir = d.getVar('UNPACKDIR')
82 srcsubdir = d.getVar('S') 80 srcsubdir = d.getVar('S')
83 81
84 def _move_file(src, dst): 82 # Add locally copied files to gitignore as we add back to the metadata directly
85 """Move a file. Creates all the directory components of destination path."""
86 dst_d = os.path.dirname(dst)
87 if dst_d:
88 bb.utils.mkdirhier(dst_d)
89 shutil.move(src, dst)
90
91 def _ls_tree(directory):
92 """Recursive listing of files in a directory"""
93 ret = []
94 for root, dirs, files in os.walk(directory):
95 ret.extend([os.path.relpath(os.path.join(root, fname), directory) for
96 fname in files])
97 return ret
98
99 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d)
100 # Move local source files into separate subdir
101 recipe_patches = [os.path.basename(patch) for patch in
102 oe.recipeutils.get_recipe_patches(d)]
103 local_files = oe.recipeutils.get_recipe_local_files(d) 83 local_files = oe.recipeutils.get_recipe_local_files(d)
104
105 if is_kernel_yocto:
106 for key in [f for f in local_files if f.endswith('scc')]:
107 with open(local_files[key], 'r') as sccfile:
108 for l in sccfile:
109 line = l.split()
110 if line and line[0] in ('kconf', 'patch'):
111 cfg = os.path.join(os.path.dirname(local_files[key]), line[-1])
112 if cfg not in local_files.values():
113 local_files[line[-1]] = cfg
114 shutil.copy2(cfg, workdir)
115
116 # Ignore local files with subdir={BP}
117 srcabspath = os.path.abspath(srcsubdir) 84 srcabspath = os.path.abspath(srcsubdir)
118 local_files = [fname for fname in local_files if 85 local_files = [fname for fname in local_files if
119 os.path.exists(os.path.join(workdir, fname)) and 86 os.path.exists(os.path.join(unpackdir, fname)) and
120 (srcabspath == workdir or not 87 srcabspath == unpackdir]
121 os.path.join(workdir, fname).startswith(srcabspath +
122 os.sep))]
123 if local_files: 88 if local_files:
124 for fname in local_files: 89 with open(os.path.join(tempdir, '.gitignore'), 'a+') as f:
125 _move_file(os.path.join(workdir, fname), 90 f.write('# Ignore local files, by default. Remove following lines'
126 os.path.join(tempdir, 'oe-local-files', fname)) 91 'if you want to commit the directory to Git\n')
127 with open(os.path.join(tempdir, 'oe-local-files', '.gitignore'), 92 for fname in local_files:
128 'w') as f: 93 f.write('%s\n' % fname)
129 f.write('# Ignore local files, by default. Remove this file ' 94
130 'if you want to commit the directory to Git\n*\n') 95 if os.path.dirname(srcsubdir) != workdir:
131
132 if srcsubdir == workdir:
133 # Find non-patch non-local sources that were "unpacked" to srctree
134 # directory
135 src_files = [fname for fname in _ls_tree(workdir) if
136 os.path.basename(fname) not in recipe_patches]
137 srcsubdir = d.getVar('DEVTOOL_PATCH_SRCDIR')
138 # Move source files to S
139 for path in src_files:
140 _move_file(os.path.join(workdir, path),
141 os.path.join(srcsubdir, path))
142 elif os.path.dirname(srcsubdir) != workdir:
143 # Handle if S is set to a subdirectory of the source 96 # Handle if S is set to a subdirectory of the source
144 srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0]) 97 srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
145 98
@@ -164,11 +117,6 @@ python devtool_post_unpack() {
164 f.write(srcsubdir) 117 f.write(srcsubdir)
165} 118}
166 119
167python devtool_pre_patch() {
168 if d.getVar('S') == d.getVar('WORKDIR'):
169 d.setVar('S', '${DEVTOOL_PATCH_SRCDIR}')
170}
171
172python devtool_post_patch() { 120python devtool_post_patch() {
173 import shutil 121 import shutil
174 tempdir = d.getVar('DEVTOOL_TEMPDIR') 122 tempdir = d.getVar('DEVTOOL_TEMPDIR')
diff --git a/meta/lib/oeqa/selftest/cases/devtool.py b/meta/lib/oeqa/selftest/cases/devtool.py
index 8ce1c65a38..c8bf7d9e44 100644
--- a/meta/lib/oeqa/selftest/cases/devtool.py
+++ b/meta/lib/oeqa/selftest/cases/devtool.py
@@ -879,13 +879,8 @@ class DevtoolModifyTests(DevtoolBase):
879 self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) 879 self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)
880 self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') 880 self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
881 result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) 881 result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
882 srcfile = os.path.join(tempdir, 'oe-local-files/share/dot.bashrc') 882 srcfile = os.path.join(tempdir, 'share/dot.bashrc')
883 srclink = os.path.join(tempdir, 'share/dot.bashrc')
884 self.assertExists(srcfile, 'Extracted source could not be found') 883 self.assertExists(srcfile, 'Extracted source could not be found')
885 if os.path.islink(srclink) and os.path.exists(srclink) and os.path.samefile(srcfile, srclink):
886 correct_symlink = True
887 self.assertTrue(correct_symlink, 'Source symlink to oe-local-files is broken')
888
889 matches = glob.glob(os.path.join(self.workspacedir, 'appends', '%s_*.bbappend' % testrecipe)) 884 matches = glob.glob(os.path.join(self.workspacedir, 'appends', '%s_*.bbappend' % testrecipe))
890 self.assertTrue(matches, 'bbappend not created') 885 self.assertTrue(matches, 'bbappend not created')
891 # Test devtool status 886 # Test devtool status
@@ -1278,7 +1273,7 @@ class DevtoolUpdateTests(DevtoolBase):
1278 with open(bbappendfile, 'r') as f: 1273 with open(bbappendfile, 'r') as f:
1279 self.assertEqual(expectedlines, f.readlines()) 1274 self.assertEqual(expectedlines, f.readlines())
1280 # Drop new commit and check patch gets deleted 1275 # Drop new commit and check patch gets deleted
1281 result = runCmd('git reset HEAD^', cwd=tempsrcdir) 1276 result = runCmd('git reset HEAD^ --hard', cwd=tempsrcdir)
1282 result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir)) 1277 result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))
1283 self.assertNotExists(patchfile, 'Patch file not deleted') 1278 self.assertNotExists(patchfile, 'Patch file not deleted')
1284 expectedlines2 = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 1279 expectedlines2 = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n',
@@ -1287,6 +1282,7 @@ class DevtoolUpdateTests(DevtoolBase):
1287 self.assertEqual(expectedlines2, f.readlines()) 1282 self.assertEqual(expectedlines2, f.readlines())
1288 # Put commit back and check we can run it if layer isn't in bblayers.conf 1283 # Put commit back and check we can run it if layer isn't in bblayers.conf
1289 os.remove(bbappendfile) 1284 os.remove(bbappendfile)
1285 result = runCmd("sed 's!\\(#define VERSION\\W*\"[^\"]*\\)\"!\\1-custom\"!' -i ReadMe.c", cwd=tempsrcdir)
1290 result = runCmd('git commit -a -m "Add our custom version"', cwd=tempsrcdir) 1286 result = runCmd('git commit -a -m "Add our custom version"', cwd=tempsrcdir)
1291 result = runCmd('bitbake-layers remove-layer %s' % templayerdir, cwd=self.builddir) 1287 result = runCmd('bitbake-layers remove-layer %s' % templayerdir, cwd=self.builddir)
1292 result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir)) 1288 result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))
@@ -1361,7 +1357,7 @@ class DevtoolUpdateTests(DevtoolBase):
1361 with open(bbappendfile, 'r') as f: 1357 with open(bbappendfile, 'r') as f:
1362 self.assertEqual(expectedlines, set(f.readlines())) 1358 self.assertEqual(expectedlines, set(f.readlines()))
1363 # Drop new commit and check SRCREV changes 1359 # Drop new commit and check SRCREV changes
1364 result = runCmd('git reset HEAD^', cwd=tempsrcdir) 1360 result = runCmd('git reset HEAD^ --hard', cwd=tempsrcdir)
1365 result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir)) 1361 result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir))
1366 self.assertNotExists(os.path.join(appenddir, testrecipe), 'Patch directory should not be created') 1362 self.assertNotExists(os.path.join(appenddir, testrecipe), 'Patch directory should not be created')
1367 result = runCmd('git rev-parse HEAD', cwd=tempsrcdir) 1363 result = runCmd('git rev-parse HEAD', cwd=tempsrcdir)
@@ -1373,6 +1369,7 @@ class DevtoolUpdateTests(DevtoolBase):
1373 self.assertEqual(expectedlines, set(f.readlines())) 1369 self.assertEqual(expectedlines, set(f.readlines()))
1374 # Put commit back and check we can run it if layer isn't in bblayers.conf 1370 # Put commit back and check we can run it if layer isn't in bblayers.conf
1375 os.remove(bbappendfile) 1371 os.remove(bbappendfile)
1372 result = runCmd('echo "# Additional line" >> Makefile.am', cwd=tempsrcdir)
1376 result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir) 1373 result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir)
1377 result = runCmd('bitbake-layers remove-layer %s' % templayerdir, cwd=self.builddir) 1374 result = runCmd('bitbake-layers remove-layer %s' % templayerdir, cwd=self.builddir)
1378 result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir)) 1375 result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir))
@@ -1404,11 +1401,12 @@ class DevtoolUpdateTests(DevtoolBase):
1404 # Try building just to ensure we haven't broken that 1401 # Try building just to ensure we haven't broken that
1405 bitbake("%s" % testrecipe) 1402 bitbake("%s" % testrecipe)
1406 # Edit / commit local source 1403 # Edit / commit local source
1407 runCmd('echo "/* Foobar */" >> oe-local-files/makedevs.c', cwd=tempdir) 1404 runCmd('echo "/* Foobar */" >> makedevs.c', cwd=tempdir)
1408 runCmd('echo "Foo" > oe-local-files/new-local', cwd=tempdir) 1405 runCmd('echo "Foo" > new-local', cwd=tempdir)
1409 runCmd('echo "Bar" > new-file', cwd=tempdir) 1406 runCmd('echo "Bar" > new-file', cwd=tempdir)
1410 runCmd('git add new-file', cwd=tempdir) 1407 runCmd('git add new-file', cwd=tempdir)
1411 runCmd('git commit -m "Add new file"', cwd=tempdir) 1408 runCmd('git commit -m "Add new file"', cwd=tempdir)
1409 runCmd('git add new-local', cwd=tempdir)
1412 runCmd('devtool update-recipe %s' % testrecipe) 1410 runCmd('devtool update-recipe %s' % testrecipe)
1413 expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), 1411 expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)),
1414 (' M', '.*/makedevs/makedevs.c$'), 1412 (' M', '.*/makedevs/makedevs.c$'),
@@ -1434,8 +1432,8 @@ class DevtoolUpdateTests(DevtoolBase):
1434 self.assertExists(local_file, 'File makedevs.c not created') 1432 self.assertExists(local_file, 'File makedevs.c not created')
1435 self.assertExists(patchfile, 'File new_local not created') 1433 self.assertExists(patchfile, 'File new_local not created')
1436 1434
1437 def test_devtool_update_recipe_local_files_2(self): 1435 def _test_devtool_update_recipe_local_files_2(self):
1438 """Check local source files support when oe-local-files is in Git""" 1436 """Check local source files support when editing local files in Git"""
1439 testrecipe = 'devtool-test-local' 1437 testrecipe = 'devtool-test-local'
1440 recipefile = get_bb_var('FILE', testrecipe) 1438 recipefile = get_bb_var('FILE', testrecipe)
1441 recipedir = os.path.dirname(recipefile) 1439 recipedir = os.path.dirname(recipefile)
@@ -1450,17 +1448,13 @@ class DevtoolUpdateTests(DevtoolBase):
1450 result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) 1448 result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
1451 # Check git repo 1449 # Check git repo
1452 self._check_src_repo(tempdir) 1450 self._check_src_repo(tempdir)
1453 # Add oe-local-files to Git
1454 runCmd('rm oe-local-files/.gitignore', cwd=tempdir)
1455 runCmd('git add oe-local-files', cwd=tempdir)
1456 runCmd('git commit -m "Add local sources"', cwd=tempdir)
1457 # Edit / commit local sources 1451 # Edit / commit local sources
1458 runCmd('echo "# Foobar" >> oe-local-files/file1', cwd=tempdir) 1452 runCmd('echo "# Foobar" >> file1', cwd=tempdir)
1459 runCmd('git commit -am "Edit existing file"', cwd=tempdir) 1453 runCmd('git commit -am "Edit existing file"', cwd=tempdir)
1460 runCmd('git rm oe-local-files/file2', cwd=tempdir) 1454 runCmd('git rm file2', cwd=tempdir)
1461 runCmd('git commit -m"Remove file"', cwd=tempdir) 1455 runCmd('git commit -m"Remove file"', cwd=tempdir)
1462 runCmd('echo "Foo" > oe-local-files/new-local', cwd=tempdir) 1456 runCmd('echo "Foo" > new-local', cwd=tempdir)
1463 runCmd('git add oe-local-files/new-local', cwd=tempdir) 1457 runCmd('git add new-local', cwd=tempdir)
1464 runCmd('git commit -m "Add new local file"', cwd=tempdir) 1458 runCmd('git commit -m "Add new local file"', cwd=tempdir)
1465 runCmd('echo "Gar" > new-file', cwd=tempdir) 1459 runCmd('echo "Gar" > new-file', cwd=tempdir)
1466 runCmd('git add new-file', cwd=tempdir) 1460 runCmd('git add new-file', cwd=tempdir)
@@ -1469,7 +1463,7 @@ class DevtoolUpdateTests(DevtoolBase):
1469 os.path.dirname(recipefile)) 1463 os.path.dirname(recipefile))
1470 # Checkout unmodified file to working copy -> devtool should still pick 1464 # Checkout unmodified file to working copy -> devtool should still pick
1471 # the modified version from HEAD 1465 # the modified version from HEAD
1472 runCmd('git checkout HEAD^ -- oe-local-files/file1', cwd=tempdir) 1466 runCmd('git checkout HEAD^ -- file1', cwd=tempdir)
1473 runCmd('devtool update-recipe %s' % testrecipe) 1467 runCmd('devtool update-recipe %s' % testrecipe)
1474 expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), 1468 expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)),
1475 (' M', '.*/file1$'), 1469 (' M', '.*/file1$'),
@@ -1544,7 +1538,7 @@ class DevtoolUpdateTests(DevtoolBase):
1544 # (don't bother with cleaning the recipe on teardown, we won't be building it) 1538 # (don't bother with cleaning the recipe on teardown, we won't be building it)
1545 result = runCmd('devtool modify %s' % testrecipe) 1539 result = runCmd('devtool modify %s' % testrecipe)
1546 # Modify one file 1540 # Modify one file
1547 runCmd('echo "Another line" >> file2', cwd=os.path.join(self.workspacedir, 'sources', testrecipe, 'oe-local-files')) 1541 runCmd('echo "Another line" >> file2', cwd=os.path.join(self.workspacedir, 'sources', testrecipe))
1548 self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) 1542 self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))
1549 result = runCmd('devtool update-recipe %s' % testrecipe) 1543 result = runCmd('devtool update-recipe %s' % testrecipe)
1550 expected_status = [(' M', '.*/%s/file2$' % testrecipe)] 1544 expected_status = [(' M', '.*/%s/file2$' % testrecipe)]
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 05161942b7..1d0fe13788 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -387,6 +387,19 @@ def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
387 ret[split[3]] = split[0:3] 387 ret[split[3]] = split[0:3]
388 return ret 388 return ret
389 389
390def _git_modified(repodir):
391 """List the difference between HEAD and the index"""
392 import bb
393 cmd = ['git', 'status', '--porcelain']
394 out, _ = bb.process.run(cmd, cwd=repodir)
395 ret = []
396 if out:
397 for line in out.split("\n"):
398 if line and not line.startswith('??'):
399 ret.append(line[3:])
400 return ret
401
402
390def _git_exclude_path(srctree, path): 403def _git_exclude_path(srctree, path):
391 """Return pathspec (list of paths) that excludes certain path""" 404 """Return pathspec (list of paths) that excludes certain path"""
392 # NOTE: "Filtering out" files/paths in this way is not entirely reliable - 405 # NOTE: "Filtering out" files/paths in this way is not entirely reliable -
@@ -460,32 +473,6 @@ def sync(args, config, basepath, workspace):
460 finally: 473 finally:
461 tinfoil.shutdown() 474 tinfoil.shutdown()
462 475
463def symlink_oelocal_files_srctree(rd, srctree):
464 import oe.patch
465 if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')):
466 # If recipe extracts to ${WORKDIR}, symlink the files into the srctree
467 # (otherwise the recipe won't build as expected)
468 local_files_dir = os.path.join(srctree, 'oe-local-files')
469 addfiles = []
470 for root, _, files in os.walk(local_files_dir):
471 relpth = os.path.relpath(root, local_files_dir)
472 if relpth != '.':
473 bb.utils.mkdirhier(os.path.join(srctree, relpth))
474 for fn in files:
475 if fn == '.gitignore':
476 continue
477 destpth = os.path.join(srctree, relpth, fn)
478 if os.path.exists(destpth):
479 os.unlink(destpth)
480 if relpth != '.':
481 back_relpth = os.path.relpath(local_files_dir, root)
482 os.symlink('%s/oe-local-files/%s/%s' % (back_relpth, relpth, fn), destpth)
483 else:
484 os.symlink('oe-local-files/%s' % fn, destpth)
485 addfiles.append(os.path.join(relpth, fn))
486 if addfiles:
487 oe.patch.GitApplyTree.commitIgnored("Add local file symlinks", dir=srctree, files=addfiles, d=rd)
488
489def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): 476def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False):
490 """Extract sources of a recipe""" 477 """Extract sources of a recipe"""
491 import oe.recipeutils 478 import oe.recipeutils
@@ -657,9 +644,6 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
657 elif not os.path.exists(workshareddir): 644 elif not os.path.exists(workshareddir):
658 oe.path.copyhardlinktree(srcsubdir, workshareddir) 645 oe.path.copyhardlinktree(srcsubdir, workshareddir)
659 646
660 tempdir_localdir = os.path.join(tempdir, 'oe-local-files')
661 srctree_localdir = os.path.join(srctree, 'oe-local-files')
662
663 if sync: 647 if sync:
664 try: 648 try:
665 logger.info('Backing up current %s branch as branch: %s.bak' % (devbranch, devbranch)) 649 logger.info('Backing up current %s branch as branch: %s.bak' % (devbranch, devbranch))
@@ -674,29 +658,8 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
674 except bb.process.ExecutionError as e: 658 except bb.process.ExecutionError as e:
675 raise DevtoolError("Error when syncing source files to local checkout: %s" % str(e)) 659 raise DevtoolError("Error when syncing source files to local checkout: %s" % str(e))
676 660
677 # Move the oe-local-files directory to srctree.
678 # As oe-local-files is not part of the constructed git tree,
679 # removing it directly during the synchronization might surprise
680 # the user. Instead, we move it to oe-local-files.bak and remind
681 # the user in the log message.
682 if os.path.exists(srctree_localdir + '.bak'):
683 shutil.rmtree(srctree_localdir + '.bak')
684
685 if os.path.exists(srctree_localdir):
686 logger.info('Backing up current local file directory %s' % srctree_localdir)
687 shutil.move(srctree_localdir, srctree_localdir + '.bak')
688
689 if os.path.exists(tempdir_localdir):
690 logger.info('Syncing local source files to srctree...')
691 shutil.copytree(tempdir_localdir, srctree_localdir)
692 else: 661 else:
693 # Move oe-local-files directory to srctree
694 if os.path.exists(tempdir_localdir):
695 logger.info('Adding local source files to srctree...')
696 shutil.move(tempdir_localdir, srcsubdir)
697
698 shutil.move(srcsubdir, srctree) 662 shutil.move(srcsubdir, srctree)
699 symlink_oelocal_files_srctree(d, srctree)
700 663
701 if is_kernel_yocto: 664 if is_kernel_yocto:
702 logger.info('Copying kernel config to srctree') 665 logger.info('Copying kernel config to srctree')
@@ -852,34 +815,22 @@ def modify(args, config, basepath, workspace):
852 if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): 815 if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch):
853 oe.path.copyhardlinktree(srcdir, srctree) 816 oe.path.copyhardlinktree(srcdir, srctree)
854 workdir = rd.getVar('WORKDIR') 817 workdir = rd.getVar('WORKDIR')
818 unpackdir = rd.getVar('UNPACKDIR')
855 srcsubdir = rd.getVar('S') 819 srcsubdir = rd.getVar('S')
856 localfilesdir = os.path.join(srctree, 'oe-local-files') 820 localfilesdir = os.path.join(srctree, 'oe-local-files')
857 # Move local source files into separate subdir
858 recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)]
859 local_files = oe.recipeutils.get_recipe_local_files(rd)
860 821
861 for key in local_files.copy(): 822 # Add locally copied files to gitignore as we add back to the metadata directly
862 if key.endswith('scc'): 823 local_files = oe.recipeutils.get_recipe_local_files(rd)
863 sccfile = open(local_files[key], 'r')
864 for l in sccfile:
865 line = l.split()
866 if line and line[0] in ('kconf', 'patch'):
867 cfg = os.path.join(os.path.dirname(local_files[key]), line[-1])
868 if not cfg in local_files.values():
869 local_files[line[-1]] = cfg
870 shutil.copy2(cfg, workdir)
871 sccfile.close()
872
873 # Ignore local files with subdir={BP}
874 srcabspath = os.path.abspath(srcsubdir) 824 srcabspath = os.path.abspath(srcsubdir)
875 local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and (srcabspath == workdir or not os.path.join(workdir, fname).startswith(srcabspath + os.sep))] 825 local_files = [fname for fname in local_files if
826 os.path.exists(os.path.join(unpackdir, fname)) and
827 srcabspath == unpackdir]
876 if local_files: 828 if local_files:
877 for fname in local_files: 829 with open(os.path.join(srctree, '.gitignore'), 'a+') as f:
878 _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname)) 830 f.write('# Ignore local files, by default. Remove following lines'
879 with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f: 831 'if you want to commit the directory to Git\n')
880 f.write('# Ignore local files, by default. Remove this file if you want to commit the directory to Git\n*\n') 832 for fname in local_files:
881 833 f.write('%s\n' % fname)
882 symlink_oelocal_files_srctree(rd, srctree)
883 834
884 task = 'do_configure' 835 task = 'do_configure'
885 res = tinfoil.build_targets(pn, task, handle_events=True) 836 res = tinfoil.build_targets(pn, task, handle_events=True)
@@ -1478,6 +1429,7 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
1478 # Instead they are directly copied over the original source files (in 1429 # Instead they are directly copied over the original source files (in
1479 # recipe space). 1430 # recipe space).
1480 existing_files = oe.recipeutils.get_recipe_local_files(rd) 1431 existing_files = oe.recipeutils.get_recipe_local_files(rd)
1432
1481 new_set = None 1433 new_set = None
1482 updated = OrderedDict() 1434 updated = OrderedDict()
1483 added = OrderedDict() 1435 added = OrderedDict()
@@ -1494,24 +1446,28 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
1494 if branchname.startswith(override_branch_prefix): 1446 if branchname.startswith(override_branch_prefix):
1495 return (updated, added, removed) 1447 return (updated, added, removed)
1496 1448
1497 local_files_dir = os.path.join(srctreebase, 'oe-local-files') 1449 files = _git_modified(srctree)
1498 git_files = _git_ls_tree(srctree) 1450 #if not files:
1499 if 'oe-local-files' in git_files: 1451 # files = _ls_tree(srctree)
1500 # If tracked by Git, take the files from srctree HEAD. First get 1452 for f in files:
1501 # the tree object of the directory 1453 fullfile = os.path.join(srctree, f)
1502 tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool') 1454 if os.path.exists(os.path.join(fullfile, ".git")):
1503 tree = git_files['oe-local-files'][2] 1455 # submodules handled elsewhere
1504 bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree, 1456 continue
1505 env=dict(os.environ, GIT_WORK_TREE=destdir, 1457 if f not in existing_files:
1506 GIT_INDEX_FILE=tmp_index)) 1458 added[f] = {}
1507 new_set = list(_git_ls_tree(srctree, tree, True).keys()) 1459 if os.path.isdir(os.path.join(srctree, f)):
1508 elif os.path.isdir(local_files_dir): 1460 shutil.copytree(fullfile, os.path.join(destdir, f))
1509 # If not tracked by Git, just copy from working copy 1461 else:
1510 new_set = _ls_tree(local_files_dir) 1462 shutil.copy2(fullfile, os.path.join(destdir, f))
1511 bb.process.run(['cp', '-ax', 1463 elif not os.path.exists(fullfile):
1512 os.path.join(local_files_dir, '.'), destdir]) 1464 removed[f] = existing_files[f]
1513 else: 1465 elif f in existing_files:
1514 new_set = [] 1466 updated[f] = {'path' : existing_files[f]}
1467 if os.path.isdir(os.path.join(srctree, f)):
1468 shutil.copytree(fullfile, os.path.join(destdir, f))
1469 else:
1470 shutil.copy2(fullfile, os.path.join(destdir, f))
1515 1471
1516 # Special handling for kernel config 1472 # Special handling for kernel config
1517 if bb.data.inherits_class('kernel-yocto', rd): 1473 if bb.data.inherits_class('kernel-yocto', rd):
@@ -1519,17 +1475,14 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
1519 fragment_path = os.path.join(destdir, fragment_fn) 1475 fragment_path = os.path.join(destdir, fragment_fn)
1520 if _create_kconfig_diff(srctree, rd, fragment_path): 1476 if _create_kconfig_diff(srctree, rd, fragment_path):
1521 if os.path.exists(fragment_path): 1477 if os.path.exists(fragment_path):
1522 if fragment_fn not in new_set: 1478 if fragment_fn in removed:
1523 new_set.append(fragment_fn) 1479 del removed[fragment_fn]
1524 # Copy fragment to local-files 1480 if fragment_fn not in updated and fragment_fn not in added:
1525 if os.path.isdir(local_files_dir): 1481 added[fragment_fn] = {}
1526 shutil.copy2(fragment_path, local_files_dir)
1527 else: 1482 else:
1528 if fragment_fn in new_set: 1483 if fragment_fn in updated:
1529 new_set.remove(fragment_fn) 1484 revoved[fragment_fn] = updated[fragment_fn]
1530 # Remove fragment from local-files 1485 del updated[fragment_fn]
1531 if os.path.exists(os.path.join(local_files_dir, fragment_fn)):
1532 os.unlink(os.path.join(local_files_dir, fragment_fn))
1533 1486
1534 # Special handling for cml1, ccmake, etc bbclasses that generated 1487 # Special handling for cml1, ccmake, etc bbclasses that generated
1535 # configuration fragment files that are consumed as source files 1488 # configuration fragment files that are consumed as source files
@@ -1537,42 +1490,13 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
1537 if bb.data.inherits_class(frag_class, rd): 1490 if bb.data.inherits_class(frag_class, rd):
1538 srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name) 1491 srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name)
1539 if os.path.exists(srcpath): 1492 if os.path.exists(srcpath):
1540 if frag_name not in new_set: 1493 if frag_name in removed:
1541 new_set.append(frag_name) 1494 del removed[frag_name]
1495 if frag_name not in updated:
1496 added[frag_name] = {}
1542 # copy fragment into destdir 1497 # copy fragment into destdir
1543 shutil.copy2(srcpath, destdir) 1498 shutil.copy2(srcpath, destdir)
1544 # copy fragment into local files if exists 1499
1545 if os.path.isdir(local_files_dir):
1546 shutil.copy2(srcpath, local_files_dir)
1547
1548 if new_set is not None:
1549 for fname in new_set:
1550 if fname in existing_files:
1551 origpath = existing_files.pop(fname)
1552 workpath = os.path.join(local_files_dir, fname)
1553 if not filecmp.cmp(origpath, workpath):
1554 updated[fname] = {'path' : origpath}
1555 elif fname != '.gitignore':
1556 added[fname] = {}
1557
1558 workdir = rd.getVar('WORKDIR')
1559 s = rd.getVar('S')
1560 if not s.endswith(os.sep):
1561 s += os.sep
1562
1563 if workdir != s:
1564 # Handle files where subdir= was specified
1565 for fname in list(existing_files.keys()):
1566 # FIXME handle both subdir starting with BP and not?
1567 fworkpath = os.path.join(workdir, fname)
1568 if fworkpath.startswith(s):
1569 fpath = os.path.join(srctree, os.path.relpath(fworkpath, s))
1570 if os.path.exists(fpath):
1571 origpath = existing_files.pop(fname)
1572 if not filecmp.cmp(origpath, fpath):
1573 updated[fpath] = {'path' : origpath}
1574
1575 removed = existing_files
1576 return (updated, added, removed) 1500 return (updated, added, removed)
1577 1501
1578 1502