summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul Barker <pbarker@konsulko.com>2021-01-19 16:26:09 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2021-01-20 22:46:18 +0000
commit36575a949399953f98fc1cfda4a3dcb5a295b02c (patch)
tree54377cbf928c456b68554e7f0351595caaf262a3
parentf85a4a1462bc2105f7c62f2b09c1d092c7677ada (diff)
downloadpoky-36575a949399953f98fc1cfda4a3dcb5a295b02c.tar.gz
wic: Copy rootfs dir if fstab needs updating
By default, wic updates the /etc/fstab in the rootfs to include details of additional partitions described in the selected wks file. If this modification is performed in place, other tasks which create an image file from the rootfs directory (e.g. do_image_tar and do_image_ext4) will pick up the modified fstab file which would not be appropriate for those images as they do not include the additional partitions described in the wks file. wic does undo modifications to the fstab file once it has finished creating the filesystem image, however this leaves open a race condition if one of the other tasks reads the contents of the fstab file from the rootfs directory between the point where wic modifies the fstab file and the point where wic restores the files original content. This could be solved by adding a lockfile for tasks which use the rootfs directory to ensure that no other such task is reading the rootfs directory while do_image_wic is running. This would serialize several do_image_* tasks and result in slower builds, especially for large images. Another drawback of this solution is that it is hard to selectively optimise - adding lockfiles to do_image_* tasks would result in these tasks always being serialized even if no fstab modification will take place. An alternative solution is to copy the rootfs directory when fstab needs to be modified. The code to do this in wic already exists as it is needed when including or excluding content in the rootfs. This still results in an impact on build times but the copy uses hardlinks if possible (so little data is actually copied) and we can make selective optimisations to improve things. The rootfs copy will only take place if fstab modification is required (or if it was already needed to include or exclude rootfs content). We can also follow up with further optimisations after this commit. So this second solution is chosen. Fixes [Yocto #13994] (From OE-Core rev: ce682a73b7447652f898ce1d1d0416a456df5416) Signed-off-by: Paul Barker <pbarker@konsulko.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--scripts/lib/wic/partition.py5
-rw-r--r--scripts/lib/wic/plugins/imager/direct.py36
-rw-r--r--scripts/lib/wic/plugins/source/rootfs.py17
3 files changed, 29 insertions, 29 deletions
diff --git a/scripts/lib/wic/partition.py b/scripts/lib/wic/partition.py
index 286c7867cb..f59eceb23d 100644
--- a/scripts/lib/wic/partition.py
+++ b/scripts/lib/wic/partition.py
@@ -54,6 +54,7 @@ class Partition():
54 self.uuid = args.uuid 54 self.uuid = args.uuid
55 self.fsuuid = args.fsuuid 55 self.fsuuid = args.fsuuid
56 self.type = args.type 56 self.type = args.type
57 self.updated_fstab_path = None
57 58
58 self.lineno = lineno 59 self.lineno = lineno
59 self.source_file = "" 60 self.source_file = ""
@@ -118,11 +119,13 @@ class Partition():
118 return self.fixed_size if self.fixed_size else self.size 119 return self.fixed_size if self.fixed_size else self.size
119 120
120 def prepare(self, creator, cr_workdir, oe_builddir, rootfs_dir, 121 def prepare(self, creator, cr_workdir, oe_builddir, rootfs_dir,
121 bootimg_dir, kernel_dir, native_sysroot): 122 bootimg_dir, kernel_dir, native_sysroot, updated_fstab_path):
122 """ 123 """
123 Prepare content for individual partitions, depending on 124 Prepare content for individual partitions, depending on
124 partition command parameters. 125 partition command parameters.
125 """ 126 """
127 self.updated_fstab_path = updated_fstab_path
128
126 if not self.source: 129 if not self.source:
127 if not self.size and not self.fixed_size: 130 if not self.size and not self.fixed_size:
128 raise WicError("The %s partition has a size of zero. Please " 131 raise WicError("The %s partition has a size of zero. Please "
diff --git a/scripts/lib/wic/plugins/imager/direct.py b/scripts/lib/wic/plugins/imager/direct.py
index f107e60089..7e1c1c03ab 100644
--- a/scripts/lib/wic/plugins/imager/direct.py
+++ b/scripts/lib/wic/plugins/imager/direct.py
@@ -58,7 +58,7 @@ class DirectPlugin(ImagerPlugin):
58 self.compressor = options.compressor 58 self.compressor = options.compressor
59 self.bmap = options.bmap 59 self.bmap = options.bmap
60 self.no_fstab_update = options.no_fstab_update 60 self.no_fstab_update = options.no_fstab_update
61 self.original_fstab = None 61 self.updated_fstab_path = None
62 62
63 self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0], 63 self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0],
64 strftime("%Y%m%d%H%M")) 64 strftime("%Y%m%d%H%M"))
@@ -100,11 +100,8 @@ class DirectPlugin(ImagerPlugin):
100 finally: 100 finally:
101 self.cleanup() 101 self.cleanup()
102 102
103 def _write_fstab(self, image_rootfs): 103 def update_fstab(self, image_rootfs):
104 """overriden to generate fstab (temporarily) in rootfs. This is called 104 """Assume partition order same as in wks"""
105 from _create, make sure it doesn't get called from
106 BaseImage.create()
107 """
108 if not image_rootfs: 105 if not image_rootfs:
109 return 106 return
110 107
@@ -114,18 +111,9 @@ class DirectPlugin(ImagerPlugin):
114 111
115 with open(fstab_path) as fstab: 112 with open(fstab_path) as fstab:
116 fstab_lines = fstab.readlines() 113 fstab_lines = fstab.readlines()
117 self.original_fstab = fstab_lines.copy()
118
119 if self._update_fstab(fstab_lines, self.parts):
120 with open(fstab_path, "w") as fstab:
121 fstab.writelines(fstab_lines)
122 else:
123 self.original_fstab = None
124 114
125 def _update_fstab(self, fstab_lines, parts):
126 """Assume partition order same as in wks"""
127 updated = False 115 updated = False
128 for part in parts: 116 for part in self.parts:
129 if not part.realnum or not part.mountpoint \ 117 if not part.realnum or not part.mountpoint \
130 or part.mountpoint == "/": 118 or part.mountpoint == "/":
131 continue 119 continue
@@ -154,7 +142,10 @@ class DirectPlugin(ImagerPlugin):
154 fstab_lines.append(line) 142 fstab_lines.append(line)
155 updated = True 143 updated = True
156 144
157 return updated 145 if updated:
146 self.updated_fstab_path = os.path.join(self.workdir, "fstab")
147 with open(self.updated_fstab_path, "w") as f:
148 f.writelines(fstab_lines)
158 149
159 def _full_path(self, path, name, extention): 150 def _full_path(self, path, name, extention):
160 """ Construct full file path to a file we generate. """ 151 """ Construct full file path to a file we generate. """
@@ -170,7 +161,7 @@ class DirectPlugin(ImagerPlugin):
170 a partitioned image. 161 a partitioned image.
171 """ 162 """
172 if not self.no_fstab_update: 163 if not self.no_fstab_update:
173 self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR")) 164 self.update_fstab(self.rootfs_dir.get("ROOTFS_DIR"))
174 165
175 for part in self.parts: 166 for part in self.parts:
176 # get rootfs size from bitbake variable if it's not set in .ks file 167 # get rootfs size from bitbake variable if it's not set in .ks file
@@ -283,12 +274,6 @@ class DirectPlugin(ImagerPlugin):
283 if os.path.isfile(path): 274 if os.path.isfile(path):
284 shutil.move(path, os.path.join(self.outdir, fname)) 275 shutil.move(path, os.path.join(self.outdir, fname))
285 276
286 #Restore original fstab
287 if self.original_fstab:
288 fstab_path = self.rootfs_dir.get("ROOTFS_DIR") + "/etc/fstab"
289 with open(fstab_path, "w") as fstab:
290 fstab.writelines(self.original_fstab)
291
292 # remove work directory 277 # remove work directory
293 shutil.rmtree(self.workdir, ignore_errors=True) 278 shutil.rmtree(self.workdir, ignore_errors=True)
294 279
@@ -368,7 +353,8 @@ class PartitionedImage():
368 # sizes before we can add them and do the layout. 353 # sizes before we can add them and do the layout.
369 part.prepare(imager, imager.workdir, imager.oe_builddir, 354 part.prepare(imager, imager.workdir, imager.oe_builddir,
370 imager.rootfs_dir, imager.bootimg_dir, 355 imager.rootfs_dir, imager.bootimg_dir,
371 imager.kernel_dir, imager.native_sysroot) 356 imager.kernel_dir, imager.native_sysroot,
357 imager.updated_fstab_path)
372 358
373 # Converting kB to sectors for parted 359 # Converting kB to sectors for parted
374 part.size_sec = part.disk_size * 1024 // self.sector_size 360 part.size_sec = part.disk_size * 1024 // self.sector_size
diff --git a/scripts/lib/wic/plugins/source/rootfs.py b/scripts/lib/wic/plugins/source/rootfs.py
index afb1eb9202..6fd415af5b 100644
--- a/scripts/lib/wic/plugins/source/rootfs.py
+++ b/scripts/lib/wic/plugins/source/rootfs.py
@@ -103,9 +103,9 @@ class RootfsPlugin(SourcePlugin):
103 new_rootfs = None 103 new_rootfs = None
104 new_pseudo = None 104 new_pseudo = None
105 # Handle excluded paths. 105 # Handle excluded paths.
106 if part.exclude_path or part.include_path or part.change_directory: 106 if part.exclude_path or part.include_path or part.change_directory or part.updated_fstab_path:
107 # We need a new rootfs directory we can delete files from. Copy to 107 # We need a new rootfs directory we can safely modify without
108 # workdir. 108 # interfering with other tasks. Copy to workdir.
109 new_rootfs = os.path.realpath(os.path.join(cr_workdir, "rootfs%d" % part.lineno)) 109 new_rootfs = os.path.realpath(os.path.join(cr_workdir, "rootfs%d" % part.lineno))
110 110
111 if os.path.lexists(new_rootfs): 111 if os.path.lexists(new_rootfs):
@@ -214,6 +214,17 @@ class RootfsPlugin(SourcePlugin):
214 rm_cmd = "rm -rf %s" % (full_path) 214 rm_cmd = "rm -rf %s" % (full_path)
215 exec_native_cmd(rm_cmd, native_sysroot, pseudo) 215 exec_native_cmd(rm_cmd, native_sysroot, pseudo)
216 216
217 has_fstab = os.path.exists(os.path.join(new_rootfs, "etc/fstab"))
218 if part.updated_fstab_path and has_fstab:
219 fstab_path = os.path.join(new_rootfs, "etc/fstab")
220 # Assume that fstab should always be owned by root with fixed permissions
221 install_cmd = "install -m 0644 %s %s" % (part.updated_fstab_path, fstab_path)
222 if new_pseudo:
223 pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo)
224 else:
225 pseudo = None
226 exec_native_cmd(install_cmd, native_sysroot, pseudo)
227
217 part.prepare_rootfs(cr_workdir, oe_builddir, 228 part.prepare_rootfs(cr_workdir, oe_builddir,
218 new_rootfs or part.rootfs_dir, native_sysroot, 229 new_rootfs or part.rootfs_dir, native_sysroot,
219 pseudo_dir = new_pseudo or pseudo_dir) 230 pseudo_dir = new_pseudo or pseudo_dir)