summaryrefslogtreecommitdiffstats
path: root/meta/recipes-devtools
diff options
context:
space:
mode:
authorRoss Burton <ross.burton@intel.com>2018-01-26 10:44:35 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-01-29 08:49:50 +0000
commita889ece9247e5237b800e0750b02603ba8f5e001 (patch)
treeabd37a05cdaf12579362ccddb9044cbeced8c38c /meta/recipes-devtools
parent0db712bd244c5eb94e1ff43eb06c0d84848799d2 (diff)
downloadpoky-a889ece9247e5237b800e0750b02603ba8f5e001.tar.gz
python: fix parse dependencies
Adding a file-checksums flag for the manifest to do_split_packages doesn't achieve anything as do_split_packages isn't a task. Changing this to tha task do_package shows that the path is wrong, but we also know that as the manifest is in SRC_URI any changes to it would result in a rebuild anyway, so this line can be deleted. However there is a problem of the recipe not being reparsed when it needs to be, if the JSON has changed. The main bitbake process can hash the recipe and use stale data from the cache as it hasn't considered the manifest file changing. This results in non-determinism warnings when the worker parses the recipe again and comes to a different hash (as the manifest has changed, so the packaging changed). Solve this by calling bb.parse.mark_dependency() to declare the dependency on the manifest. (From OE-Core rev: a321b28c8dafc9775f465ce7c0f6bcbe8ccc2945) Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/recipes-devtools')
-rw-r--r--meta/recipes-devtools/python/python3_3.5.3.bb45
-rw-r--r--meta/recipes-devtools/python/python_2.7.13.bb38
2 files changed, 39 insertions, 44 deletions
diff --git a/meta/recipes-devtools/python/python3_3.5.3.bb b/meta/recipes-devtools/python/python3_3.5.3.bb
index 970ba63445..3ae0db2b8b 100644
--- a/meta/recipes-devtools/python/python3_3.5.3.bb
+++ b/meta/recipes-devtools/python/python3_3.5.3.bb
@@ -228,21 +228,22 @@ FILES_${PN}-man = "${datadir}/man"
228BBCLASSEXTEND = "nativesdk" 228BBCLASSEXTEND = "nativesdk"
229 229
230RPROVIDES_${PN} += "${PN}-modules" 230RPROVIDES_${PN} += "${PN}-modules"
231 231
232# We want bytecode precompiled .py files (.pyc's) by default 232# We want bytecode precompiled .py files (.pyc's) by default
233# but the user may set it on their own conf 233# but the user may set it on their own conf
234
235INCLUDE_PYCS ?= "1" 234INCLUDE_PYCS ?= "1"
236 235
237python(){ 236python(){
237 import json
238 238
239 pythondir = d.getVar('THISDIR',True) 239 filename = os.path.join(d.getVar('THISDIR'), 'python3', 'python3-manifest.json')
240 # This python changes the datastore based on the contents of a file, so mark
241 # that dependency.
242 bb.parse.mark_dependency(d, filename)
240 243
241 # Read JSON manifest 244 with open(filename) as manifest_file:
242 import json
243 with open(pythondir+'/python3/python3-manifest.json') as manifest_file:
244 python_manifest=json.load(manifest_file) 245 python_manifest=json.load(manifest_file)
245 246
246 include_pycs = d.getVar('INCLUDE_PYCS') 247 include_pycs = d.getVar('INCLUDE_PYCS')
247 248
248 packages = d.getVar('PACKAGES').split() 249 packages = d.getVar('PACKAGES').split()
@@ -282,28 +283,24 @@ python(){
282 d.setVar('PACKAGES', ' '.join(packages)) 283 d.setVar('PACKAGES', ' '.join(packages))
283 d.setVar('ALLOW_EMPTY_${PN}-modules', '1') 284 d.setVar('ALLOW_EMPTY_${PN}-modules', '1')
284} 285}
285do_split_packages[file-checksums] += "${THISDIR}/python/python3-manifest.json:True"
286
287
288 286
289# Files needed to create a new manifest 287# Files needed to create a new manifest
290SRC_URI += "file://create_manifest3.py file://get_module_deps3.py file://python3-manifest.json" 288SRC_URI += "file://create_manifest3.py file://get_module_deps3.py file://python3-manifest.json"
291 289
292do_create_manifest() { 290do_create_manifest() {
293 291 # This task should be run with every new release of Python.
294# This task should be run with every new release of Python. 292 # We must ensure that PACKAGECONFIG enables everything when creating
295# We must ensure that PACKAGECONFIG enables everything when creating 293 # a new manifest, this is to base our new manifest on a complete
296# a new manifest, this is to base our new manifest on a complete 294 # native python build, containing all dependencies, otherwise the task
297# native python build, containing all dependencies, otherwise the task 295 # wont be able to find the required files.
298# wont be able to find the required files. 296 # e.g. BerkeleyDB is an optional build dependency so it may or may not
299# e.g. BerkeleyDB is an optional build dependency so it may or may not 297 # be present, we must ensure it is.
300# be present, we must ensure it is. 298
301 299 cd ${WORKDIR}
302cd ${WORKDIR} 300 # This needs to be executed by python-native and NOT by HOST's python
303# This needs to be executed by python-native and NOT by HOST's python 301 nativepython3 create_manifest3.py
304nativepython3 create_manifest3.py 302 cp python3-manifest.json.new ${THISDIR}/python3/python3-manifest.json
305cp python3-manifest.json.new ${THISDIR}/python3/python3-manifest.json 303}
306}
307 304
308# bitbake python -c create_manifest 305# bitbake python -c create_manifest
309addtask do_create_manifest 306addtask do_create_manifest
diff --git a/meta/recipes-devtools/python/python_2.7.13.bb b/meta/recipes-devtools/python/python_2.7.13.bb
index dbafb955f9..337c7447bb 100644
--- a/meta/recipes-devtools/python/python_2.7.13.bb
+++ b/meta/recipes-devtools/python/python_2.7.13.bb
@@ -202,12 +202,14 @@ RPROVIDES_${PN} += "${PN}-modules"
202INCLUDE_PYCS ?= "1" 202INCLUDE_PYCS ?= "1"
203 203
204python(){ 204python(){
205 import json
205 206
206 pythondir = d.getVar('THISDIR',True) 207 filename = os.path.join(d.getVar('THISDIR'), 'python', 'python2-manifest.json')
208 # This python changes the datastore based on the contents of a file, so mark
209 # that dependency.
210 bb.parse.mark_dependency(d, filename)
207 211
208 # Read JSON manifest 212 with open(filename) as manifest_file:
209 import json
210 with open(pythondir+'/python/python2-manifest.json') as manifest_file:
211 python_manifest=json.load(manifest_file) 213 python_manifest=json.load(manifest_file)
212 214
213 include_pycs = d.getVar('INCLUDE_PYCS') 215 include_pycs = d.getVar('INCLUDE_PYCS')
@@ -215,7 +217,6 @@ python(){
215 packages = d.getVar('PACKAGES').split() 217 packages = d.getVar('PACKAGES').split()
216 pn = d.getVar('PN') 218 pn = d.getVar('PN')
217 219
218
219 newpackages=[] 220 newpackages=[]
220 221
221 for key in python_manifest: 222 for key in python_manifest:
@@ -250,25 +251,22 @@ python(){
250 d.setVar('ALLOW_EMPTY_${PN}-modules', '1') 251 d.setVar('ALLOW_EMPTY_${PN}-modules', '1')
251} 252}
252 253
253do_split_packages[file-checksums] += "${THISDIR}/python/python2-manifest.json:True"
254
255# Files needed to create a new manifest 254# Files needed to create a new manifest
256SRC_URI += "file://create_manifest2.py file://get_module_deps2.py file://python2-manifest.json" 255SRC_URI += "file://create_manifest2.py file://get_module_deps2.py file://python2-manifest.json"
257 256
258do_create_manifest() { 257do_create_manifest() {
259 258 # This task should be run with every new release of Python.
260# This task should be run with every new release of Python. 259 # We must ensure that PACKAGECONFIG enables everything when creating
261# We must ensure that PACKAGECONFIG enables everything when creating 260 # a new manifest, this is to base our new manifest on a complete
262# a new manifest, this is to base our new manifest on a complete 261 # native python build, containing all dependencies, otherwise the task
263# native python build, containing all dependencies, otherwise the task 262 # wont be able to find the required files.
264# wont be able to find the required files. 263 # e.g. BerkeleyDB is an optional build dependency so it may or may not
265# e.g. BerkeleyDB is an optional build dependency so it may or may not 264 # be present, we must ensure it is.
266# be present, we must ensure it is. 265
267 266 cd ${WORKDIR}
268cd ${WORKDIR} 267 # This needs to be executed by python-native and NOT by HOST's python
269# This needs to be executed by python-native and NOT by HOST's python 268 nativepython create_manifest2.py
270nativepython create_manifest2.py 269 cp python2-manifest.json.new ${THISDIR}/python/python2-manifest.json
271cp python2-manifest.json.new ${THISDIR}/python/python2-manifest.json
272} 270}
273 271
274# bitbake python -c create_manifest 272# bitbake python -c create_manifest