summaryrefslogtreecommitdiffstats
path: root/scripts/lib/devtool/sdk.py
diff options
context:
space:
mode:
authorQi.Chen@windriver.com <Qi.Chen@windriver.com>2015-09-07 13:42:27 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-09-09 14:27:43 +0100
commitee428e27fed0da8651184cf3ac4a0dd39c4e9d9d (patch)
tree80af781a558304f74aa4995142a57c8310b5de71 /scripts/lib/devtool/sdk.py
parent2aaa59ed8e3bb72385e962ec3594da709b18fad8 (diff)
downloadpoky-ee428e27fed0da8651184cf3ac4a0dd39c4e9d9d.tar.gz
devtool: add mechanism for updating extensible SDK
Enable updating the installed extensible SDK from a local or remote server, avoiding the need to install it again from scratch when updating. (This assumes that the updated SDK has been built and then published somewhere using the oe-publish-sdk script beforehand.) This plugin is only enabled when devtool is used within the extensible SDK since it doesn't make sense to use it next to a normal install of the build system. E.g. devtool sdk-update /mnt/sdk-repo/ devtool sdk-update http://mysdkhost/sdk (From OE-Core rev: 32cbd4c57fc8ca097a18929fc404c07322ef36dd) Signed-off-by: Chen Qi <Qi.Chen@windriver.com> Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/devtool/sdk.py')
-rw-r--r--scripts/lib/devtool/sdk.py197
1 files changed, 197 insertions, 0 deletions
diff --git a/scripts/lib/devtool/sdk.py b/scripts/lib/devtool/sdk.py
new file mode 100644
index 0000000000..2f416b36f5
--- /dev/null
+++ b/scripts/lib/devtool/sdk.py
@@ -0,0 +1,197 @@
1# Development tool - sdk-update command plugin
2
3import os
4import subprocess
5import logging
6import glob
7import shutil
8import errno
9import sys
10from devtool import exec_build_env_command, setup_tinfoil, DevtoolError
11
12logger = logging.getLogger('devtool')
13
14def plugin_init(pluginlist):
15 """Plugin initialization"""
16 pass
17
18def parse_locked_sigs(sigfile_path):
19 """Return <pn:task>:<hash> dictionary"""
20 sig_dict = {}
21 with open(sigfile_path) as f:
22 lines = f.readlines()
23 for line in lines:
24 if ':' in line:
25 taskkey, _, hashval = line.rpartition(':')
26 sig_dict[taskkey.strip()] = hashval.split()[0]
27 return sig_dict
28
29def generate_update_dict(sigfile_new, sigfile_old):
30 """Return a dict containing <pn:task>:<hash> which indicates what need to be updated"""
31 update_dict = {}
32 sigdict_new = parse_locked_sigs(sigfile_new)
33 sigdict_old = parse_locked_sigs(sigfile_old)
34 for k in sigdict_new:
35 if k not in sigdict_old:
36 update_dict[k] = sigdict_new[k]
37 continue
38 if sigdict_new[k] != sigdict_old[k]:
39 update_dict[k] = sigdict_new[k]
40 continue
41 return update_dict
42
43def get_sstate_objects(update_dict, newsdk_path):
44 """Return a list containing sstate objects which are to be installed"""
45 sstate_objects = []
46 # Ensure newsdk_path points to an extensible SDK
47 sstate_dir = os.path.join(newsdk_path, 'sstate-cache')
48 if not os.path.exists(sstate_dir):
49 logger.error("sstate-cache directory not found under %s" % newsdk_path)
50 raise
51 for k in update_dict:
52 files = set()
53 hashval = update_dict[k]
54 p = sstate_dir + '/' + hashval[:2] + '/*' + hashval + '*.tgz'
55 files |= set(glob.glob(p))
56 p = sstate_dir + '/*/' + hashval[:2] + '/*' + hashval + '*.tgz'
57 files |= set(glob.glob(p))
58 files = list(files)
59 if len(files) == 1:
60 sstate_objects.extend(files)
61 elif len(files) > 1:
62 logger.error("More than one matching sstate object found for %s" % hashval)
63
64 return sstate_objects
65
66def mkdir(d):
67 try:
68 os.makedirs(d)
69 except OSError as e:
70 if e.errno != errno.EEXIST:
71 raise e
72
73def install_sstate_objects(sstate_objects, src_sdk, dest_sdk):
74 """Install sstate objects into destination SDK"""
75 sstate_dir = os.path.join(dest_sdk, 'sstate-cache')
76 if not os.path.exists(sstate_dir):
77 logger.error("Missing sstate-cache directory in %s, it might not be an extensible SDK." % dest_sdk)
78 raise
79 for sb in sstate_objects:
80 dst = sb.replace(src_sdk, dest_sdk)
81 destdir = os.path.dirname(dst)
82 mkdir(destdir)
83 logger.debug("Copying %s to %s" % (sb, dst))
84 shutil.copy(sb, dst)
85
86def sdk_update(args, config, basepath, workspace):
87 # Fetch locked-sigs.inc file from remote/local destination
88 from ConfigParser import NoSectionError
89 updateserver = args.updateserver
90 if not updateserver:
91 try:
92 updateserver = config.get('SDK', 'updateserver', None)
93 except NoSectionError:
94 pass
95 if not updateserver:
96 raise DevtoolError("Update server not specified in config file, you must specify it on the command line")
97 logger.debug("updateserver: %s" % args.updateserver)
98
99 # Make sure we are using sdk-update from within SDK
100 logger.debug("basepath = %s" % basepath)
101 old_locked_sig_file_path = os.path.join(basepath, 'conf/locked-sigs.inc')
102 if not os.path.exists(old_locked_sig_file_path):
103 logger.error("Not using devtool's sdk-update command from within an extensible SDK. Please specify correct basepath via --basepath option")
104 return -1
105 else:
106 logger.debug("Found conf/locked-sigs.inc in %s" % basepath)
107
108 if ':' in args.updateserver:
109 is_remote = True
110 else:
111 is_remote = False
112
113 if not is_remote:
114 # devtool sdk-update /local/path/to/latest/sdk
115 new_locked_sig_file_path = os.path.join(args.updateserver, 'conf/locked-sigs.inc')
116 if not os.path.exists(new_locked_sig_file_path):
117 logger.error("%s doesn't exist or is not an extensible SDK" % args.updateserver)
118 return -1
119 else:
120 logger.debug("Found conf/locked-sigs.inc in %s" % args.updateserver)
121 update_dict = generate_update_dict(new_locked_sig_file_path, old_locked_sig_file_path)
122 logger.debug("update_dict = %s" % update_dict)
123 sstate_objects = get_sstate_objects(update_dict, args.updateserver)
124 logger.debug("sstate_objects = %s" % sstate_objects)
125 if len(sstate_objects) == 0:
126 logger.info("No need to update.")
127 return 0
128 logger.info("Installing sstate objects into %s", basepath)
129 install_sstate_objects(sstate_objects, args.updateserver.rstrip('/'), basepath)
130 logger.info("Updating configuration files")
131 new_conf_dir = os.path.join(args.updateserver, 'conf')
132 old_conf_dir = os.path.join(basepath, 'conf')
133 shutil.rmtree(old_conf_dir)
134 shutil.copytree(new_conf_dir, old_conf_dir)
135 logger.info("Updating layers")
136 new_layers_dir = os.path.join(args.updateserver, 'layers')
137 old_layers_dir = os.path.join(basepath, 'layers')
138 shutil.rmtree(old_layers_dir)
139 shutil.copytree(new_layers_dir, old_layers_dir)
140 else:
141 # devtool sdk-update http://myhost/sdk
142 tmpsdk_dir = '/tmp/sdk-ext'
143 if os.path.exists(tmpsdk_dir):
144 shutil.rmtree(tmpsdk_dir)
145 os.makedirs(tmpsdk_dir)
146 os.makedirs(os.path.join(tmpsdk_dir, 'conf'))
147 # Fetch locked-sigs.inc from update server
148 ret = subprocess.call("wget -q -O - %s/conf/locked-sigs.inc > %s/locked-sigs.inc" % (args.updateserver, os.path.join(tmpsdk_dir, 'conf')), shell=True)
149 if ret != 0:
150 logger.error("Fetching conf/locked-sigs.inc from %s to %s/locked-sigs.inc failed" % (args.updateserver, os.path.join(tmpsdk_dir, 'conf')))
151 return ret
152 else:
153 logger.info("Fetching conf/locked-sigs.inc from %s to %s/locked-sigs.inc succeeded" % (args.updateserver, os.path.join(tmpsdk_dir, 'conf')))
154 new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf/locked-sigs.inc')
155 update_dict = generate_update_dict(new_locked_sig_file_path, old_locked_sig_file_path)
156 logger.debug("update_dict = %s" % update_dict)
157 if len(update_dict) == 0:
158 logger.info("No need to update.")
159 return 0
160 # Update metadata
161 logger.debug("Updating meta data via git ...")
162 # Try using 'git pull', if failed, use 'git clone'
163 if os.path.exists(os.path.join(basepath, 'layers/.git')):
164 ret = subprocess.call("cd layers && git pull", shell=True)
165 else:
166 ret = -1
167 if ret != 0:
168 ret = subprocess.call("rm -rf layers && git clone %s/layers" % args.updateserver, shell=True)
169 if ret != 0:
170 logger.error("Updating meta data via git failed")
171 return ret
172 logger.debug("Updating conf files ...")
173 conf_files = ['local.conf', 'bblayers.conf', 'devtool.conf', 'work-config.inc', 'locked-sigs.inc']
174 for conf in conf_files:
175 ret = subprocess.call("wget -q -O - %s/conf/%s > conf/%s" % (args.updateserver, conf, conf), shell=True)
176 if ret != 0:
177 logger.error("Update %s failed" % conf)
178 return ret
179 with open(os.path.join(basepath, 'conf/local.conf'), 'a') as f:
180 f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % args.updateserver)
181
182 # Run bitbake command for the whole SDK
183 sdk_targets = config.get('SDK', 'sdk_targets')
184 logger.info("Executing 'bitbake %s' ... (This may take some time.)" % sdk_targets)
185 try:
186 exec_build_env_command(config.init_path, basepath, 'bitbake %s' % sdk_targets)
187 except:
188 logger.error('bitbake %s failed' % sdk_targets)
189 return -1
190 return 0
191
192def register_commands(subparsers, context):
193 """Register devtool subcommands from the sdk plugin"""
194 if context.fixed_setup:
195 parser_sdk = subparsers.add_parser('sdk-update', help='Update SDK components from a nominated location')
196 parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from', nargs='?')
197 parser_sdk.set_defaults(func=sdk_update)