summaryrefslogtreecommitdiffstats
path: root/meta/classes
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-01-30 17:42:25 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-02-17 10:19:59 +0000
commit050aab7f45ed13e01da5ff760e8292c091b27ee6 (patch)
treea7875a813949ae72afb056246feb217361c63ab6 /meta/classes
parent554379870fab3552fad774c065c02fa295f02b08 (diff)
downloadpoky-050aab7f45ed13e01da5ff760e8292c091b27ee6.tar.gz
package: Rework debug source file handling
Currently we parallel process the files we install running dwarfsrcfiles over each one in parallel threads but requiring a lock to write the results to one file. This is not ideal for performance and means we can't then use per file data for other purposes such as source code license processing. Rework the code so that the list of source files is generated per installed file and is reusable. The code still generates a null separated debugsources.list file since this is used by a shell pipeline but it no longer needs locking. (From OE-Core rev: 95de93988eb725c14102f642ebabff3920ae194f) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/classes')
-rw-r--r--meta/classes/package.bbclass50
1 files changed, 27 insertions, 23 deletions
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 7dd1b09a87..749c7d9ea1 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -344,7 +344,7 @@ def parse_debugsources_from_dwarfsrcfiles_output(dwarfsrcfiles_output):
344 344
345 return debugfiles.keys() 345 return debugfiles.keys()
346 346
347def append_source_info(file, sourcefile, d, fatal=True): 347def source_info(file, d, fatal=True):
348 import subprocess 348 import subprocess
349 349
350 cmd = ["dwarfsrcfiles", file] 350 cmd = ["dwarfsrcfiles", file]
@@ -363,22 +363,15 @@ def append_source_info(file, sourcefile, d, fatal=True):
363 bb.note(msg) 363 bb.note(msg)
364 364
365 debugsources = parse_debugsources_from_dwarfsrcfiles_output(output) 365 debugsources = parse_debugsources_from_dwarfsrcfiles_output(output)
366 # filenames are null-separated - this is an artefact of the previous use
367 # of rpm's debugedit, which was writing them out that way, and the code elsewhere
368 # is still assuming that.
369 debuglistoutput = '\0'.join(debugsources) + '\0'
370 lf = bb.utils.lockfile(sourcefile + ".lock")
371 with open(sourcefile, 'a') as sf:
372 sf.write(debuglistoutput)
373 bb.utils.unlockfile(lf)
374 366
367 return list(debugsources)
375 368
376def splitdebuginfo(file, dvar, debugdir, debuglibdir, debugappend, debugsrcdir, sourcefile, d): 369def splitdebuginfo(file, dvar, debugdir, debuglibdir, debugappend, debugsrcdir, d):
377 # Function to split a single file into two components, one is the stripped 370 # Function to split a single file into two components, one is the stripped
378 # target system binary, the other contains any debugging information. The 371 # target system binary, the other contains any debugging information. The
379 # two files are linked to reference each other. 372 # two files are linked to reference each other.
380 # 373 #
381 # sourcefile is also generated containing a list of debugsources 374 # return a mapping of files:debugsources
382 375
383 import stat 376 import stat
384 import subprocess 377 import subprocess
@@ -386,6 +379,7 @@ def splitdebuginfo(file, dvar, debugdir, debuglibdir, debugappend, debugsrcdir,
386 src = file[len(dvar):] 379 src = file[len(dvar):]
387 dest = debuglibdir + os.path.dirname(src) + debugdir + "/" + os.path.basename(src) + debugappend 380 dest = debuglibdir + os.path.dirname(src) + debugdir + "/" + os.path.basename(src) + debugappend
388 debugfile = dvar + dest 381 debugfile = dvar + dest
382 sources = []
389 383
390 # Split the file... 384 # Split the file...
391 bb.utils.mkdirhier(os.path.dirname(debugfile)) 385 bb.utils.mkdirhier(os.path.dirname(debugfile))
@@ -397,7 +391,7 @@ def splitdebuginfo(file, dvar, debugdir, debuglibdir, debugappend, debugsrcdir,
397 391
398 # We ignore kernel modules, we don't generate debug info files. 392 # We ignore kernel modules, we don't generate debug info files.
399 if file.find("/lib/modules/") != -1 and file.endswith(".ko"): 393 if file.find("/lib/modules/") != -1 and file.endswith(".ko"):
400 return 1 394 return (file, sources)
401 395
402 newmode = None 396 newmode = None
403 if not os.access(file, os.W_OK) or os.access(file, os.R_OK): 397 if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
@@ -407,7 +401,7 @@ def splitdebuginfo(file, dvar, debugdir, debuglibdir, debugappend, debugsrcdir,
407 401
408 # We need to extract the debug src information here... 402 # We need to extract the debug src information here...
409 if debugsrcdir: 403 if debugsrcdir:
410 append_source_info(file, sourcefile, d) 404 sources = source_info(file, d)
411 405
412 bb.utils.mkdirhier(os.path.dirname(debugfile)) 406 bb.utils.mkdirhier(os.path.dirname(debugfile))
413 407
@@ -419,17 +413,26 @@ def splitdebuginfo(file, dvar, debugdir, debuglibdir, debugappend, debugsrcdir,
419 if newmode: 413 if newmode:
420 os.chmod(file, origmode) 414 os.chmod(file, origmode)
421 415
422 return 0 416 return (file, sources)
423 417
424def copydebugsources(debugsrcdir, d): 418def copydebugsources(debugsrcdir, sources, d):
425 # The debug src information written out to sourcefile is further processed 419 # The debug src information written out to sourcefile is further processed
426 # and copied to the destination here. 420 # and copied to the destination here.
427 421
428 import stat 422 import stat
429 import subprocess 423 import subprocess
430 424
431 sourcefile = d.expand("${WORKDIR}/debugsources.list") 425 if debugsrcdir and sources:
432 if debugsrcdir and os.path.isfile(sourcefile): 426 sourcefile = d.expand("${WORKDIR}/debugsources.list")
427 bb.utils.remove(sourcefile)
428
429 # filenames are null-separated - this is an artefact of the previous use
430 # of rpm's debugedit, which was writing them out that way, and the code elsewhere
431 # is still assuming that.
432 debuglistoutput = '\0'.join(sources) + '\0'
433 with open(sourcefile, 'a') as sf:
434 sf.write(debuglistoutput)
435
433 dvar = d.getVar('PKGD') 436 dvar = d.getVar('PKGD')
434 strip = d.getVar("STRIP") 437 strip = d.getVar("STRIP")
435 objcopy = d.getVar("OBJCOPY") 438 objcopy = d.getVar("OBJCOPY")
@@ -933,9 +936,6 @@ python split_and_strip_files () {
933 debuglibdir = "" 936 debuglibdir = ""
934 debugsrcdir = "/usr/src/debug" 937 debugsrcdir = "/usr/src/debug"
935 938
936 sourcefile = d.expand("${WORKDIR}/debugsources.list")
937 bb.utils.remove(sourcefile)
938
939 # 939 #
940 # First lets figure out all of the files we may have to process ... do this only once! 940 # First lets figure out all of the files we may have to process ... do this only once!
941 # 941 #
@@ -1040,11 +1040,15 @@ python split_and_strip_files () {
1040 # First lets process debug splitting 1040 # First lets process debug splitting
1041 # 1041 #
1042 if (d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'): 1042 if (d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'):
1043 oe.utils.multiprocess_launch(splitdebuginfo, list(elffiles), d, extraargs=(dvar, debugdir, debuglibdir, debugappend, debugsrcdir, sourcefile, d)) 1043 results = oe.utils.multiprocess_launch(splitdebuginfo, list(elffiles), d, extraargs=(dvar, debugdir, debuglibdir, debugappend, debugsrcdir, d))
1044 1044
1045 if debugsrcdir and not targetos.startswith("mingw"): 1045 if debugsrcdir and not targetos.startswith("mingw"):
1046 for file in staticlibs: 1046 for file in staticlibs:
1047 append_source_info(file, sourcefile, d, fatal=False) 1047 results.extend(source_info(file, d, fatal=False))
1048
1049 sources = set()
1050 for r in results:
1051 sources.update(r[1])
1048 1052
1049 # Hardlink our debug symbols to the other hardlink copies 1053 # Hardlink our debug symbols to the other hardlink copies
1050 for ref in inodes: 1054 for ref in inodes:
@@ -1092,7 +1096,7 @@ python split_and_strip_files () {
1092 1096
1093 # Process the debugsrcdir if requested... 1097 # Process the debugsrcdir if requested...
1094 # This copies and places the referenced sources for later debugging... 1098 # This copies and places the referenced sources for later debugging...
1095 copydebugsources(debugsrcdir, d) 1099 copydebugsources(debugsrcdir, sources, d)
1096 # 1100 #
1097 # End of debug splitting 1101 # End of debug splitting
1098 # 1102 #