summaryrefslogtreecommitdiffstats
path: root/meta/classes
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2017-02-06 14:47:55 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-02-07 14:50:09 +0000
commitab78c782f73be76cff55220b582b01d173d7f711 (patch)
tree4f6974753dba7ea209ea0d196898b12158a8573a /meta/classes
parent175a4570322a4909c223b193fcb77f00bcb8362a (diff)
downloadpoky-ab78c782f73be76cff55220b582b01d173d7f711.tar.gz
staging: Improve file creation resiliance
If you abort a build mid execution of the staging extend_sysroot function there are currently races and cleanup of that function may fail. This change splits the code into separate phases so that the manifests are manipulated before files are installed, meaning we should be able to reverse actions if builds fail, crash or are interrupted. (From OE-Core rev: fc504b6b8538453d2f058d7cc69be098f52f8091) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/classes')
-rw-r--r--meta/classes/staging.bbclass88
1 files changed, 49 insertions, 39 deletions
diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass
index bd798ba9bc..0cb46bfd24 100644
--- a/meta/classes/staging.bbclass
+++ b/meta/classes/staging.bbclass
@@ -245,17 +245,9 @@ python do_populate_sysroot_setscene () {
245} 245}
246addtask do_populate_sysroot_setscene 246addtask do_populate_sysroot_setscene
247 247
248def staging_copyfile(c, target, fixme, postinsts, stagingdir, seendirs): 248def staging_copyfile(c, target, dest, postinsts, seendirs):
249 import errno 249 import errno
250 250
251 if c.endswith("/fixmepath"):
252 fixme.append(c)
253 return None
254 if c.endswith("/fixmepath.cmd"):
255 return None
256 #bb.warn(c)
257 dest = c.replace(stagingdir, "")
258 dest = target + "/" + "/".join(dest.split("/")[3:])
259 destdir = os.path.dirname(dest) 251 destdir = os.path.dirname(dest)
260 if destdir not in seendirs: 252 if destdir not in seendirs:
261 bb.utils.mkdirhier(destdir) 253 bb.utils.mkdirhier(destdir)
@@ -282,9 +274,7 @@ def staging_copyfile(c, target, fixme, postinsts, stagingdir, seendirs):
282 raise 274 raise
283 return dest 275 return dest
284 276
285def staging_copydir(c, target, stagingdir, seendirs): 277def staging_copydir(c, target, dest, seendirs):
286 dest = c.replace(stagingdir, "")
287 dest = target + "/" + "/".join(dest.split("/")[3:])
288 if dest not in seendirs: 278 if dest not in seendirs:
289 bb.utils.mkdirhier(dest) 279 bb.utils.mkdirhier(dest)
290 seendirs.add(dest) 280 seendirs.add(dest)
@@ -338,11 +328,18 @@ def staging_populate_sysroot_dir(targetsysroot, nativesysroot, native, d):
338 with open(manifest, "r") as f: 328 with open(manifest, "r") as f:
339 for l in f: 329 for l in f:
340 l = l.strip() 330 l = l.strip()
331 if l.endswith("/fixmepath"):
332 fixme.append(l)
333 continue
334 if l.endswith("/fixmepath.cmd"):
335 continue
336 dest = l.replace(stagingdir, "")
337 dest = targetdir + "/" + "/".join(dest.split("/")[3:])
341 if l.endswith("/"): 338 if l.endswith("/"):
342 staging_copydir(l, targetdir, stagingdir, seendirs) 339 staging_copydir(l, targetdir, dest, seendirs)
343 continue 340 continue
344 try: 341 try:
345 staging_copyfile(l, targetdir, fixme, postinsts, stagingdir, seendirs) 342 staging_copyfile(l, targetdir, dest, postinsts, seendirs)
346 except FileExistsError: 343 except FileExistsError:
347 continue 344 continue
348 345
@@ -365,6 +362,8 @@ def staging_populate_sysroot_dir(targetsysroot, nativesysroot, native, d):
365python extend_recipe_sysroot() { 362python extend_recipe_sysroot() {
366 import copy 363 import copy
367 import subprocess 364 import subprocess
365 import errno
366 import collections
368 367
369 taskdepdata = d.getVar("BB_TASKDEPDATA", False) 368 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
370 mytaskname = d.getVar("BB_RUNTASK") 369 mytaskname = d.getVar("BB_RUNTASK")
@@ -574,43 +573,54 @@ python extend_recipe_sysroot() {
574 if not os.path.exists(manifest): 573 if not os.path.exists(manifest):
575 bb.warn("Manifest %s not found?" % manifest) 574 bb.warn("Manifest %s not found?" % manifest)
576 else: 575 else:
577 with open(manifest, "r") as f, open(taskmanifest, 'w') as m: 576 newmanifest = collections.OrderedDict()
577 if native:
578 fm = fixme['native']
579 targetdir = recipesysrootnative
580 else:
581 fm = fixme['']
582 targetdir = destsysroot
583 with open(manifest, "r") as f:
578 manifests[dep] = manifest 584 manifests[dep] = manifest
579 for l in f: 585 for l in f:
580 l = l.strip() 586 l = l.strip()
581 if l.endswith("/"): 587 if l.endswith("/fixmepath"):
582 if native: 588 fm.append(l)
583 dest = staging_copydir(l, recipesysrootnative, stagingdir, seendirs) 589 continue
584 else: 590 if l.endswith("/fixmepath.cmd"):
585 dest = staging_copydir(l, destsysroot, stagingdir, seendirs)
586 continue 591 continue
587 if native: 592 dest = l.replace(stagingdir, "")
588 dest = staging_copyfile(l, recipesysrootnative, fixme['native'], postinsts, stagingdir, seendirs) 593 dest = targetdir + "/" + "/".join(dest.split("/")[3:])
589 else: 594 newmanifest[l] = dest
590 dest = staging_copyfile(l, destsysroot, fixme[''], postinsts, stagingdir, seendirs)
591 if dest:
592 m.write(dest.replace(workdir + "/", "") + "\n")
593 # Having multiple identical manifests in each sysroot eats diskspace so 595 # Having multiple identical manifests in each sysroot eats diskspace so
594 # create a shared pool of them. 596 # create a shared pool of them and hardlink if we can.
597 # We create the manifest in advance so that if something fails during installation,
598 # or the build is interrupted, subsequent exeuction can cleanup.
595 sharedm = sharedmanifests + "/" + os.path.basename(taskmanifest) 599 sharedm = sharedmanifests + "/" + os.path.basename(taskmanifest)
596 if not os.path.exists(sharedm): 600 if not os.path.exists(sharedm):
597 smlock = bb.utils.lockfile(sharedm + ".lock") 601 smlock = bb.utils.lockfile(sharedm + ".lock")
598 # Can race here. You'd think it just means we may not end up with all copies hardlinked to each other 602 # Can race here. You'd think it just means we may not end up with all copies hardlinked to each other
599 # but python can lose file handles so we need to do this under a lock. 603 # but python can lose file handles so we need to do this under a lock.
600 try: 604 if not os.path.exists(sharedm):
601 if not os.path.exists(sharedm): 605 with open(sharedm, 'w') as m:
602 os.rename(taskmanifest, sharedm) 606 for l in newmanifest:
603 except OSError: 607 dest = newmanifest[l]
604 pass 608 m.write(dest.replace(workdir + "/", "") + "\n")
605 bb.utils.unlockfile(smlock) 609 bb.utils.unlockfile(smlock)
606 if os.path.exists(sharedm): 610 try:
607 # If we're crossing mount points we'll not reach here.
608 if os.path.exists(taskmanifest):
609 if os.path.getsize(sharedm) != os.path.getsize(taskmanifest):
610 # Order of entries can differ, overall size shouldn't
611 raise Exception("Manifests %s and %s differ in size and shouldn't?" % (sharedm, taskmanifest))
612 os.unlink(taskmanifest)
613 os.link(sharedm, taskmanifest) 611 os.link(sharedm, taskmanifest)
612 except OSError as err:
613 if err.errno == errno.EXDEV:
614 bb.utils.copyfile(sharedm, taskmanifest)
615 else:
616 raise
617 # Finally actually install the files
618 for l in newmanifest:
619 dest = newmanifest[l]
620 if l.endswith("/"):
621 staging_copydir(l, targetdir, dest, seendirs)
622 continue
623 staging_copyfile(l, targetdir, dest, postinsts, seendirs)
614 624
615 for f in fixme: 625 for f in fixme:
616 if f == '': 626 if f == '':