summaryrefslogtreecommitdiffstats
path: root/meta
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2017-09-05 10:56:18 +1200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-09-18 11:07:29 +0100
commit10af6d86b3effc523cfa0ec49741c5b02ee2cf86 (patch)
tree64974ae0b7f5fbfaa5647618bf401b98b1d93228 /meta
parent4842db00b7aaac96f7bbdfbea5d8bcdb2ebac81b (diff)
downloadpoky-10af6d86b3effc523cfa0ec49741c5b02ee2cf86.tar.gz
devtool: rework source extraction so that dependencies are handled
Since it was first implemented, devtool's source extraction (as used by the devtool modify, extract and upgrade subcommands) ignored other recipe dependencies - so for example if you ran devtool modify on a recipe that fetches from svn or is compressed using xz then it would fail if those dependencies hadn't been built first. Now that we can execute tasks in the normal way (i.e. tinfoil.build_targets()) then we can rework it to use that. This is slightly tricky in that the source extraction needs to insert some logic in between tasks; luckily we can use a helper class that conditionally adds prefuncs to make that possible. Some side-effects / aspects of this change worth noting: * Operations are a little slower because we have to go through the task dependency graph generation and other startup processing. There's not really any way to avoid this though. * devtool extract didn't used to require a workspace, now it does because it needs to create a temporary bbappend for the recipe. (As with other commands the workspace be created on the fly if it doesn't already exist.) * I want any existing sysroot files and stamps to be left alone during extraction since we are running the tasks off to the side, and especially devtool extract should be able to be used without touching these. However, this was hampered by the automatic removal process in sstate.bbclass triggered by bb.event.ReachableStamps when the task signatures change, thus I had to introduce a way to disable this removal on a per-recipe basis (we still want it to function for any dependencies that we aren't working on). To implement this I elected to use a file written to tmp/sstate-control which gets deleted automatically after reading so that there's less chance of stale files affecting future sessions. I could have used a variable but this would have needed to be whitelisted and I'd have to have poked its value in using the setVariable command. Fixes [YOCTO #11198]. (From OE-Core rev: 830dbd66992cbb9e731b48d56fddf8f220349666) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta')
-rw-r--r--meta/classes/devtool-source.bbclass165
-rw-r--r--meta/classes/sstate.bbclass9
-rw-r--r--meta/lib/oeqa/selftest/cases/devtool.py10
3 files changed, 178 insertions, 6 deletions
diff --git a/meta/classes/devtool-source.bbclass b/meta/classes/devtool-source.bbclass
new file mode 100644
index 0000000000..8f5bc86b2e
--- /dev/null
+++ b/meta/classes/devtool-source.bbclass
@@ -0,0 +1,165 @@
1# Development tool - source extraction helper class
2#
3# NOTE: this class is intended for use by devtool and should not be
4# inherited manually.
5#
6# Copyright (C) 2014-2017 Intel Corporation
7#
8# This program is free software; you can redistribute it and/or modify
9# it under the terms of the GNU General Public License version 2 as
10# published by the Free Software Foundation.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License along
18# with this program; if not, write to the Free Software Foundation, Inc.,
19# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21
22DEVTOOL_TEMPDIR ?= ""
23DEVTOOL_PATCH_SRCDIR = "${DEVTOOL_TEMPDIR}/patchworkdir"
24
25
26python() {
27 tempdir = d.getVar('DEVTOOL_TEMPDIR')
28
29 if not tempdir:
30 bb.fatal('devtool-source class is for internal use by devtool only')
31
32 # Make a subdir so we guard against WORKDIR==S
33 workdir = os.path.join(tempdir, 'workdir')
34 d.setVar('WORKDIR', workdir)
35 if not d.getVar('S').startswith(workdir):
36 # Usually a shared workdir recipe (kernel, gcc)
37 # Try to set a reasonable default
38 if bb.data.inherits_class('kernel', d):
39 d.setVar('S', '${WORKDIR}/source')
40 else:
41 d.setVar('S', '${WORKDIR}/%s' % os.path.basename(d.getVar('S')))
42 if bb.data.inherits_class('kernel', d):
43 # We don't want to move the source to STAGING_KERNEL_DIR here
44 d.setVar('STAGING_KERNEL_DIR', '${S}')
45
46 d.setVar('STAMPS_DIR', os.path.join(tempdir, 'stamps'))
47 d.setVar('T', os.path.join(tempdir, 'temp'))
48
49 # Hook in pre/postfuncs
50 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d)
51 if is_kernel_yocto:
52 unpacktask = 'do_kernel_checkout'
53 d.appendVarFlag('do_configure', 'postfuncs', ' devtool_post_configure')
54 else:
55 unpacktask = 'do_unpack'
56 d.appendVarFlag(unpacktask, 'postfuncs', ' devtool_post_unpack')
57 d.prependVarFlag('do_patch', 'prefuncs', ' devtool_pre_patch')
58 d.appendVarFlag('do_patch', 'postfuncs', ' devtool_post_patch')
59
60 # NOTE: in order for the patch stuff to be fully functional,
61 # PATCHTOOL and PATCH_COMMIT_FUNCTIONS need to be set; we can't
62 # do that here because we can't guarantee the order of the anonymous
63 # functions, so it gets done in the bbappend we create.
64}
65
66
67python devtool_post_unpack() {
68 import oe.recipeutils
69 import shutil
70 sys.path.insert(0, os.path.join(d.getVar('COREBASE'), 'scripts', 'lib'))
71 import scriptutils
72 from devtool import setup_git_repo
73
74 tempdir = d.getVar('DEVTOOL_TEMPDIR')
75 workdir = d.getVar('WORKDIR')
76 srcsubdir = d.getVar('S')
77
78 def _move_file(src, dst):
79 """Move a file. Creates all the directory components of destination path."""
80 dst_d = os.path.dirname(dst)
81 if dst_d:
82 bb.utils.mkdirhier(dst_d)
83 shutil.move(src, dst)
84
85 def _ls_tree(directory):
86 """Recursive listing of files in a directory"""
87 ret = []
88 for root, dirs, files in os.walk(directory):
89 ret.extend([os.path.relpath(os.path.join(root, fname), directory) for
90 fname in files])
91 return ret
92
93 # Move local source files into separate subdir
94 recipe_patches = [os.path.basename(patch) for patch in
95 oe.recipeutils.get_recipe_patches(d)]
96 local_files = oe.recipeutils.get_recipe_local_files(d)
97
98 # Ignore local files with subdir={BP}
99 srcabspath = os.path.abspath(srcsubdir)
100 local_files = [fname for fname in local_files if
101 os.path.exists(os.path.join(workdir, fname)) and
102 (srcabspath == workdir or not
103 os.path.join(workdir, fname).startswith(srcabspath +
104 os.sep))]
105 if local_files:
106 for fname in local_files:
107 _move_file(os.path.join(workdir, fname),
108 os.path.join(tempdir, 'oe-local-files', fname))
109 with open(os.path.join(tempdir, 'oe-local-files', '.gitignore'),
110 'w') as f:
111 f.write('# Ignore local files, by default. Remove this file '
112 'if you want to commit the directory to Git\n*\n')
113
114 if srcsubdir == workdir:
115 # Find non-patch non-local sources that were "unpacked" to srctree
116 # directory
117 src_files = [fname for fname in _ls_tree(workdir) if
118 os.path.basename(fname) not in recipe_patches]
119 srcsubdir = d.getVar('DEVTOOL_PATCH_SRCDIR')
120 # Move source files to S
121 for path in src_files:
122 _move_file(os.path.join(workdir, path),
123 os.path.join(srcsubdir, path))
124 elif os.path.dirname(srcsubdir) != workdir:
125 # Handle if S is set to a subdirectory of the source
126 srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
127
128 scriptutils.git_convert_standalone_clone(srcsubdir)
129
130 # Make sure that srcsubdir exists
131 bb.utils.mkdirhier(srcsubdir)
132 if not os.listdir(srcsubdir):
133 bb.warn("No source unpacked to S - either the %s recipe "
134 "doesn't use any source or the correct source "
135 "directory could not be determined" % d.getVar('PN'))
136
137 devbranch = d.getVar('DEVTOOL_DEVBRANCH')
138 setup_git_repo(srcsubdir, d.getVar('PV'), devbranch, d=d)
139
140 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
141 initial_rev = stdout.rstrip()
142 with open(os.path.join(tempdir, 'initial_rev'), 'w') as f:
143 f.write(initial_rev)
144
145 with open(os.path.join(tempdir, 'srcsubdir'), 'w') as f:
146 f.write(srcsubdir)
147}
148
149python devtool_pre_patch() {
150 if d.getVar('S') == d.getVar('WORKDIR'):
151 d.setVar('S', '${DEVTOOL_PATCH_SRCDIR}')
152}
153
154python devtool_post_patch() {
155 tempdir = d.getVar('DEVTOOL_TEMPDIR')
156 with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f:
157 srcsubdir = f.read()
158 bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
159}
160
161python devtool_post_configure() {
162 import shutil
163 tempdir = d.getVar('DEVTOOL_TEMPDIR')
164 shutil.copy2(os.path.join(d.getVar('B'), '.config'), tempdir)
165}
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 6af0d388bc..2a54993d1d 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -1015,6 +1015,11 @@ python sstate_eventhandler2() {
1015 d = e.data 1015 d = e.data
1016 stamps = e.stamps.values() 1016 stamps = e.stamps.values()
1017 removeworkdir = (d.getVar("SSTATE_PRUNE_OBSOLETEWORKDIR", False) == "1") 1017 removeworkdir = (d.getVar("SSTATE_PRUNE_OBSOLETEWORKDIR", False) == "1")
1018 preservestampfile = d.expand('${SSTATE_MANIFESTS}/preserve-stamps')
1019 preservestamps = []
1020 if os.path.exists(preservestampfile):
1021 with open(preservestampfile, 'r') as f:
1022 preservestamps = f.readlines()
1018 seen = [] 1023 seen = []
1019 for a in d.getVar("SSTATE_ARCHS").split(): 1024 for a in d.getVar("SSTATE_ARCHS").split():
1020 toremove = [] 1025 toremove = []
@@ -1025,7 +1030,7 @@ python sstate_eventhandler2() {
1025 lines = f.readlines() 1030 lines = f.readlines()
1026 for l in lines: 1031 for l in lines:
1027 (stamp, manifest, workdir) = l.split() 1032 (stamp, manifest, workdir) = l.split()
1028 if stamp not in stamps: 1033 if stamp not in stamps and stamp not in preservestamps:
1029 toremove.append(l) 1034 toremove.append(l)
1030 if stamp not in seen: 1035 if stamp not in seen:
1031 bb.debug(2, "Stamp %s is not reachable, removing related manifests" % stamp) 1036 bb.debug(2, "Stamp %s is not reachable, removing related manifests" % stamp)
@@ -1047,4 +1052,6 @@ python sstate_eventhandler2() {
1047 with open(i, "w") as f: 1052 with open(i, "w") as f:
1048 for l in lines: 1053 for l in lines:
1049 f.write(l) 1054 f.write(l)
1055 if preservestamps:
1056 os.remove(preservestampfile)
1050} 1057}
diff --git a/meta/lib/oeqa/selftest/cases/devtool.py b/meta/lib/oeqa/selftest/cases/devtool.py
index 1dfef599e6..3c537ee071 100644
--- a/meta/lib/oeqa/selftest/cases/devtool.py
+++ b/meta/lib/oeqa/selftest/cases/devtool.py
@@ -1073,7 +1073,7 @@ class DevtoolTests(DevtoolBase):
1073 1073
1074 @OETestID(1628) 1074 @OETestID(1628)
1075 def test_devtool_update_recipe_local_files_subdir(self): 1075 def test_devtool_update_recipe_local_files_subdir(self):
1076 # Try devtool extract on a recipe that has a file with subdir= set in 1076 # Try devtool update-recipe on a recipe that has a file with subdir= set in
1077 # SRC_URI such that it overwrites a file that was in an archive that 1077 # SRC_URI such that it overwrites a file that was in an archive that
1078 # was also in SRC_URI 1078 # was also in SRC_URI
1079 # First, modify the recipe 1079 # First, modify the recipe
@@ -1103,10 +1103,10 @@ class DevtoolTests(DevtoolBase):
1103 tempdir = tempfile.mkdtemp(prefix='devtoolqa') 1103 tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1104 # Try devtool extract 1104 # Try devtool extract
1105 self.track_for_cleanup(tempdir) 1105 self.track_for_cleanup(tempdir)
1106 self.track_for_cleanup(self.workspacedir)
1107 self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1106 result = runCmd('devtool extract matchbox-terminal %s' % tempdir) 1108 result = runCmd('devtool extract matchbox-terminal %s' % tempdir)
1107 self.assertExists(os.path.join(tempdir, 'Makefile.am'), 'Extracted source could not be found') 1109 self.assertExists(os.path.join(tempdir, 'Makefile.am'), 'Extracted source could not be found')
1108 # devtool extract shouldn't create the workspace
1109 self.assertNotExists(self.workspacedir)
1110 self._check_src_repo(tempdir) 1110 self._check_src_repo(tempdir)
1111 1111
1112 @OETestID(1379) 1112 @OETestID(1379)
@@ -1114,10 +1114,10 @@ class DevtoolTests(DevtoolBase):
1114 tempdir = tempfile.mkdtemp(prefix='devtoolqa') 1114 tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1115 # Try devtool extract 1115 # Try devtool extract
1116 self.track_for_cleanup(tempdir) 1116 self.track_for_cleanup(tempdir)
1117 self.track_for_cleanup(self.workspacedir)
1118 self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1117 result = runCmd('devtool extract virtual/make %s' % tempdir) 1119 result = runCmd('devtool extract virtual/make %s' % tempdir)
1118 self.assertExists(os.path.join(tempdir, 'Makefile.am'), 'Extracted source could not be found') 1120 self.assertExists(os.path.join(tempdir, 'Makefile.am'), 'Extracted source could not be found')
1119 # devtool extract shouldn't create the workspace
1120 self.assertNotExists(self.workspacedir)
1121 self._check_src_repo(tempdir) 1121 self._check_src_repo(tempdir)
1122 1122
1123 @OETestID(1168) 1123 @OETestID(1168)