summaryrefslogtreecommitdiffstats
path: root/scripts/lib/devtool/sdk.py
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2016-01-23 00:59:51 +1300
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-01-24 09:40:32 +0000
commit44d1a2a45c4d92f331aef1a95adb6b12743d24cd (patch)
treebd4d1c53ca62bb40c14eee04c6a40141ac01169c /scripts/lib/devtool/sdk.py
parent3360baa96bb2ebd54efaba0fb9aa9a1c9093c233 (diff)
downloadpoky-44d1a2a45c4d92f331aef1a95adb6b12743d24cd.tar.gz
devtool: sdk-update: improve SDK update process robustness
Make the following improvements to the SDK update process: * Use a manifest file with sha256sums to track files other than sstate and metadata that we need to update - e.g. conf files. This allows us to handle where files such as auto.conf may or may not be present, as well as the configuration changing without affecting task signatures - we still want the config files copied in that case rather than it saying nothing needs to be done. * Write the SSTATE_MIRRORS_append to site.conf rather than local.conf so that local.conf remains static (since we don't want to trigger an update every time). Also, If there is an SSTATE_MIRRORS value already set in the configuration we can skip this and assume it contains the needed packages. * Allow the update process to be run in any directory, don't assume we're already at the base of the SDK * Where practical, fetch remote files into a temporary location and then move them to the desired location at the end, to avoid a failed update leaving the SDK in a broken state. * Update all installed do_populate_sysroot / do_packagedata tasks instead of using the SDK targets. This ensures any item installed through dependencies after installation (e.g. when running "devtool build") won't go stale. (From OE-Core rev: 3d35631121f0e030bc8151f5c23d84008d06f44b) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/devtool/sdk.py')
-rw-r--r--scripts/lib/devtool/sdk.py118
1 files changed, 84 insertions, 34 deletions
diff --git a/scripts/lib/devtool/sdk.py b/scripts/lib/devtool/sdk.py
index 4fcd36a0df..80ea8711cc 100644
--- a/scripts/lib/devtool/sdk.py
+++ b/scripts/lib/devtool/sdk.py
@@ -75,6 +75,21 @@ def install_sstate_objects(sstate_objects, src_sdk, dest_sdk):
75 logger.debug("Copying %s to %s" % (sb, dst)) 75 logger.debug("Copying %s to %s" % (sb, dst))
76 shutil.copy(sb, dst) 76 shutil.copy(sb, dst)
77 77
78def check_manifest(fn, basepath):
79 import bb.utils
80 changedfiles = []
81 with open(fn, 'r') as f:
82 for line in f:
83 splitline = line.split()
84 if len(splitline) > 1:
85 chksum = splitline[0]
86 fpath = splitline[1]
87 curr_chksum = bb.utils.sha256_file(os.path.join(basepath, fpath))
88 if chksum != curr_chksum:
89 logger.debug('File %s changed: old csum = %s, new = %s' % (os.path.join(basepath, fpath), curr_chksum, chksum))
90 changedfiles.append(fpath)
91 return changedfiles
92
78def sdk_update(args, config, basepath, workspace): 93def sdk_update(args, config, basepath, workspace):
79 # Fetch locked-sigs.inc file from remote/local destination 94 # Fetch locked-sigs.inc file from remote/local destination
80 updateserver = args.updateserver 95 updateserver = args.updateserver
@@ -98,6 +113,18 @@ def sdk_update(args, config, basepath, workspace):
98 else: 113 else:
99 is_remote = False 114 is_remote = False
100 115
116 layers_dir = os.path.join(basepath, 'layers')
117 conf_dir = os.path.join(basepath, 'conf')
118
119 # Grab variable values
120 tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
121 try:
122 stamps_dir = tinfoil.config_data.getVar('STAMPS_DIR', True)
123 sstate_mirrors = tinfoil.config_data.getVar('SSTATE_MIRRORS', True)
124 site_conf_version = tinfoil.config_data.getVar('SITE_CONF_VERSION', True)
125 finally:
126 tinfoil.shutdown()
127
101 if not is_remote: 128 if not is_remote:
102 # devtool sdk-update /local/path/to/latest/sdk 129 # devtool sdk-update /local/path/to/latest/sdk
103 new_locked_sig_file_path = os.path.join(updateserver, 'conf/locked-sigs.inc') 130 new_locked_sig_file_path = os.path.join(updateserver, 'conf/locked-sigs.inc')
@@ -121,16 +148,14 @@ def sdk_update(args, config, basepath, workspace):
121 install_sstate_objects(sstate_objects, updateserver.rstrip('/'), basepath) 148 install_sstate_objects(sstate_objects, updateserver.rstrip('/'), basepath)
122 logger.info("Updating configuration files") 149 logger.info("Updating configuration files")
123 new_conf_dir = os.path.join(updateserver, 'conf') 150 new_conf_dir = os.path.join(updateserver, 'conf')
124 old_conf_dir = os.path.join(basepath, 'conf') 151 shutil.rmtree(conf_dir)
125 shutil.rmtree(old_conf_dir) 152 shutil.copytree(new_conf_dir, conf_dir)
126 shutil.copytree(new_conf_dir, old_conf_dir)
127 logger.info("Updating layers") 153 logger.info("Updating layers")
128 new_layers_dir = os.path.join(updateserver, 'layers') 154 new_layers_dir = os.path.join(updateserver, 'layers')
129 old_layers_dir = os.path.join(basepath, 'layers') 155 shutil.rmtree(layers_dir)
130 shutil.rmtree(old_layers_dir) 156 ret = subprocess.call("cp -a %s %s" % (new_layers_dir, layers_dir), shell=True)
131 ret = subprocess.call("cp -a %s %s" % (new_layers_dir, old_layers_dir), shell=True)
132 if ret != 0: 157 if ret != 0:
133 logger.error("Copying %s to %s failed" % (new_layers_dir, old_layers_dir)) 158 logger.error("Copying %s to %s failed" % (new_layers_dir, layers_dir))
134 return ret 159 return ret
135 else: 160 else:
136 # devtool sdk-update http://myhost/sdk 161 # devtool sdk-update http://myhost/sdk
@@ -138,50 +163,75 @@ def sdk_update(args, config, basepath, workspace):
138 try: 163 try:
139 os.makedirs(os.path.join(tmpsdk_dir, 'conf')) 164 os.makedirs(os.path.join(tmpsdk_dir, 'conf'))
140 new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf', 'locked-sigs.inc') 165 new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf', 'locked-sigs.inc')
141 # Fetch locked-sigs.inc from update server 166 # Fetch manifest from server
142 ret = subprocess.call("wget -q -O - %s/conf/locked-sigs.inc > %s" % (updateserver, new_locked_sig_file_path), shell=True) 167 tmpmanifest = os.path.join(tmpsdk_dir, 'conf', 'sdk-conf-manifest')
143 if ret != 0: 168 ret = subprocess.call("wget -q -O %s %s/conf/sdk-conf-manifest" % (tmpmanifest, updateserver), shell=True)
144 logger.error("Fetching conf/locked-sigs.inc from %s to %s failed" % (updateserver, new_locked_sig_file_path)) 169 changedfiles = check_manifest(tmpmanifest, basepath)
145 return ret 170 if not changedfiles:
146 else: 171 logger.info("Already up-to-date")
147 logger.info("Fetching conf/locked-sigs.inc from %s to %s succeeded" % (updateserver, new_locked_sig_file_path))
148 update_dict = generate_update_dict(new_locked_sig_file_path, old_locked_sig_file_path)
149 logger.debug("update_dict = %s" % update_dict)
150 if len(update_dict) == 0:
151 logger.info("No need to update.")
152 return 0 172 return 0
153 # Update metadata 173 # Update metadata
154 logger.debug("Updating meta data via git ...") 174 logger.debug("Updating metadata via git ...")
155 # Try using 'git pull', if failed, use 'git clone' 175 # Try using 'git pull', if failed, use 'git clone'
156 if os.path.exists(os.path.join(basepath, 'layers/.git')): 176 if os.path.exists(os.path.join(basepath, 'layers/.git')):
157 ret = subprocess.call("cd layers && git pull %s/layers/.git" % updateserver, shell=True) 177 ret = subprocess.call("git pull %s/layers/.git" % updateserver, shell=True, cwd=layers_dir)
158 else: 178 else:
159 ret = -1 179 ret = -1
160 if ret != 0: 180 if ret != 0:
161 ret = subprocess.call("rm -rf layers && git clone %s/layers/.git" % updateserver, shell=True) 181 ret = subprocess.call("git clone %s/layers/.git" % updateserver, shell=True, cwd=tmpsdk_dir)
162 if ret != 0: 182 if ret != 0:
163 logger.error("Updating meta data via git failed") 183 logger.error("Updating metadata via git failed")
164 return ret 184 return ret
165 logger.debug("Updating conf files ...") 185 logger.debug("Updating conf files ...")
166 conf_files = ['local.conf', 'bblayers.conf', 'devtool.conf', 'locked-sigs.inc'] 186 for changedfile in changedfiles:
167 for conf in conf_files: 187 ret = subprocess.call("wget -q -O %s %s/%s" % (changedfile, updateserver, changedfile), shell=True, cwd=tmpsdk_dir)
168 ret = subprocess.call("wget -q -O - %s/conf/%s > conf/%s" % (updateserver, conf, conf), shell=True)
169 if ret != 0: 188 if ret != 0:
170 logger.error("Update %s failed" % conf) 189 logger.error("Updating %s failed" % changedfile)
171 return ret 190 return ret
172 with open(os.path.join(basepath, 'conf/local.conf'), 'a') as f: 191
173 f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver) 192 # Ok, all is well at this point - move everything over
193 tmplayers_dir = os.path.join(tmpsdk_dir, 'layers')
194 if os.path.exists(tmplayers_dir):
195 shutil.rmtree(layers_dir)
196 shutil.move(tmplayers_dir, layers_dir)
197 for changedfile in changedfiles:
198 destfile = os.path.join(basepath, changedfile)
199 os.remove(destfile)
200 shutil.move(os.path.join(tmpsdk_dir, changedfile), destfile)
201 os.remove(os.path.join(conf_dir, 'sdk-conf-manifest'))
202 shutil.move(tmpmanifest, conf_dir)
203
204 if not sstate_mirrors:
205 with open(os.path.join(conf_dir, 'site.conf'), 'a') as f:
206 f.write('SCONF_VERSION = "%s"\n' % site_conf_version)
207 f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver)
174 finally: 208 finally:
175 shutil.rmtree(tmpsdk_dir) 209 shutil.rmtree(tmpsdk_dir)
176 210
177 if not args.skip_prepare: 211 if not args.skip_prepare:
212 # Find all potentially updateable tasks
213 sdk_update_targets = []
214 tasks = ['do_populate_sysroot', 'do_packagedata']
215 for root, _, files in os.walk(stamps_dir):
216 for fn in files:
217 if not '.sigdata.' in fn:
218 for task in tasks:
219 if '.%s.' % task in fn or '.%s_setscene.' % task in fn:
220 sdk_update_targets.append('%s:%s' % (os.path.basename(root), task))
178 # Run bitbake command for the whole SDK 221 # Run bitbake command for the whole SDK
179 sdk_update_targets = config.get('SDK', 'sdk_update_targets', config.get('SDK', 'sdk_targets'))
180 logger.info("Preparing build system... (This may take some time.)") 222 logger.info("Preparing build system... (This may take some time.)")
181 try: 223 try:
182 exec_build_env_command(config.init_path, basepath, 'bitbake %s --setscene-only' % sdk_update_targets) 224 exec_build_env_command(config.init_path, basepath, 'bitbake --setscene-only %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT)
183 except: 225 output, _ = exec_build_env_command(config.init_path, basepath, 'bitbake -n %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT)
184 logger.error('bitbake %s failed' % sdk_update_targets) 226 runlines = []
227 for line in output.splitlines():
228 if 'Running task ' in line:
229 runlines.append(line)
230 if runlines:
231 logger.error('Unexecuted tasks found in preparation log:\n %s' % '\n '.join(runlines))
232 return -1
233 except bb.process.ExecutionError as e:
234 logger.error('Preparation failed:\n%s' % e.stdout)
185 return -1 235 return -1
186 return 0 236 return 0
187 237