diff options
Diffstat (limited to 'bitbake')
-rw-r--r-- | bitbake/lib/toaster/orm/management/commands/lsupdates.py | 251 |
1 files changed, 245 insertions, 6 deletions
diff --git a/bitbake/lib/toaster/orm/management/commands/lsupdates.py b/bitbake/lib/toaster/orm/management/commands/lsupdates.py index 75e9513fca..89ee53e3dd 100644 --- a/bitbake/lib/toaster/orm/management/commands/lsupdates.py +++ b/bitbake/lib/toaster/orm/management/commands/lsupdates.py | |||
@@ -1,12 +1,251 @@ | |||
1 | from django.core.management.base import NoArgsCommand, CommandError | 1 | # |
2 | from orm.models import LayerSource | 2 | # ex:ts=4:sw=4:sts=4:et |
3 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- | ||
4 | # | ||
5 | # BitBake Toaster Implementation | ||
6 | # | ||
7 | # Copyright (C) 2016 Intel Corporation | ||
8 | # | ||
9 | # This program is free software; you can redistribute it and/or modify | ||
10 | # it under the terms of the GNU General Public License version 2 as | ||
11 | # published by the Free Software Foundation. | ||
12 | # | ||
13 | # This program is distributed in the hope that it will be useful, | ||
14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
16 | # GNU General Public License for more details. | ||
17 | # | ||
18 | # You should have received a copy of the GNU General Public License along | ||
19 | # with this program; if not, write to the Free Software Foundation, Inc., | ||
20 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | ||
21 | |||
22 | from django.core.management.base import NoArgsCommand | ||
23 | |||
24 | from orm.models import LayerSource, Layer, Release, Layer_Version | ||
25 | from orm.models import LayerVersionDependency, Machine, Recipe | ||
26 | |||
3 | import os | 27 | import os |
28 | import json | ||
29 | import logging | ||
30 | logger = logging.getLogger("toaster") | ||
31 | |||
32 | DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/" | ||
33 | |||
4 | 34 | ||
5 | class Command(NoArgsCommand): | 35 | class Command(NoArgsCommand): |
6 | args = "" | 36 | args = "" |
7 | help = "Updates locally cached information from all LayerSources" | 37 | help = "Updates locally cached information from a layerindex server" |
38 | |||
39 | def update(self): | ||
40 | """ | ||
41 | Fetches layer, recipe and machine information from a layerindex | ||
42 | server | ||
43 | """ | ||
44 | |||
45 | self.apiurl = DEFAULT_LAYERINDEX_SERVER | ||
46 | |||
47 | assert self.apiurl is not None | ||
48 | try: | ||
49 | from urllib.request import urlopen, URLError | ||
50 | from urllib.parse import urlparse | ||
51 | except ImportError: | ||
52 | from urllib2 import urlopen, URLError | ||
53 | from urlparse import urlparse | ||
54 | |||
55 | proxy_settings = os.environ.get("http_proxy", None) | ||
56 | oe_core_layer = 'openembedded-core' | ||
57 | |||
58 | def _get_json_response(apiurl=DEFAULT_LAYERINDEX_SERVER): | ||
59 | _parsedurl = urlparse(apiurl) | ||
60 | path = _parsedurl.path | ||
61 | |||
62 | # logger.debug("Fetching %s", apiurl) | ||
63 | try: | ||
64 | res = urlopen(apiurl) | ||
65 | except URLError as e: | ||
66 | raise Exception("Failed to read %s: %s" % (path, e.reason)) | ||
67 | |||
68 | return json.loads(res.read().decode('utf-8')) | ||
69 | |||
70 | # verify we can get the basic api | ||
71 | try: | ||
72 | apilinks = _get_json_response() | ||
73 | except Exception as e: | ||
74 | import traceback | ||
75 | if proxy_settings is not None: | ||
76 | logger.info("EE: Using proxy %s" % proxy_settings) | ||
77 | logger.warning("EE: could not connect to %s, skipping update:" | ||
78 | "%s\n%s" % (self.apiurl, e, traceback.format_exc())) | ||
79 | return | ||
80 | |||
81 | # update branches; only those that we already have names listed in the | ||
82 | # Releases table | ||
83 | whitelist_branch_names = [rel.branch_name | ||
84 | for rel in Release.objects.all()] | ||
85 | if len(whitelist_branch_names) == 0: | ||
86 | raise Exception("Failed to make list of branches to fetch") | ||
87 | |||
88 | logger.debug("Fetching branches") | ||
89 | |||
90 | # keep a track of the id mappings so that layer_versions can be created | ||
91 | # for these layers later on | ||
92 | li_layer_id_to_toaster_layer_id = {} | ||
93 | |||
94 | # We may need this? TODO | ||
95 | #branches_info = _get_json_response(apilinks['branches'] + | ||
96 | # "?filter=name:%s" | ||
97 | # % "OR".join(whitelist_branch_names)) | ||
98 | |||
99 | # update layers | ||
100 | layers_info = _get_json_response(apilinks['layerItems']) | ||
101 | |||
102 | for li in layers_info: | ||
103 | # Special case for the openembedded-core layer | ||
104 | if li['name'] == oe_core_layer: | ||
105 | try: | ||
106 | # If we have an existing openembedded-core for example | ||
107 | # from the toasterconf.json augment the info using the | ||
108 | # layerindex rather than duplicate it | ||
109 | oe_core_l = Layer.objects.get(name=oe_core_layer) | ||
110 | # Take ownership of the layer as now coming from the | ||
111 | # layerindex | ||
112 | oe_core_l.summary = li['summary'] | ||
113 | oe_core_l.description = li['description'] | ||
114 | oe_core_l.save() | ||
115 | li_layer_id_to_toaster_layer_id[li['id']] = oe_core_l.pk | ||
116 | continue | ||
117 | |||
118 | except Layer.DoesNotExist: | ||
119 | pass | ||
120 | |||
121 | l, created = Layer.objects.get_or_create(name=li['name']) | ||
122 | l.up_date = li['updated'] | ||
123 | l.vcs_url = li['vcs_url'] | ||
124 | l.vcs_web_url = li['vcs_web_url'] | ||
125 | l.vcs_web_tree_base_url = li['vcs_web_tree_base_url'] | ||
126 | l.vcs_web_file_base_url = li['vcs_web_file_base_url'] | ||
127 | l.summary = li['summary'] | ||
128 | l.description = li['description'] | ||
129 | l.save() | ||
130 | |||
131 | li_layer_id_to_toaster_layer_id[li['id']] = l.pk | ||
132 | |||
133 | # update layerbranches/layer_versions | ||
134 | logger.debug("Fetching layer information") | ||
135 | layerbranches_info = _get_json_response( | ||
136 | apilinks['layerBranches'] + "?filter=branch__name:%s" % | ||
137 | "OR".join(whitelist_branch_names)) | ||
138 | |||
139 | # Map Layer index layer_branch object id to | ||
140 | # layer_version toaster object id | ||
141 | li_layer_branch_id_to_toaster_lv_id = {} | ||
142 | |||
143 | for lbi in layerbranches_info: | ||
144 | |||
145 | try: | ||
146 | lv, created = Layer_Version.objects.get_or_create( | ||
147 | layer_source=LayerSource.TYPE_LAYERINDEX, | ||
148 | layer=Layer.objects.get( | ||
149 | pk=li_layer_id_to_toaster_layer_id[lbi['layer']]) | ||
150 | ) | ||
151 | except KeyError: | ||
152 | print("No such layerindex layer referenced by layerbranch %d" % | ||
153 | lbi['layer']) | ||
154 | continue | ||
155 | |||
156 | lv.up_date = lbi['updated'] | ||
157 | lv.commit = lbi['actual_branch'] | ||
158 | lv.dirpath = lbi['vcs_subdir'] | ||
159 | lv.save() | ||
160 | |||
161 | li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\ | ||
162 | lv.pk | ||
163 | |||
164 | # update layer dependencies | ||
165 | layerdependencies_info = _get_json_response( | ||
166 | apilinks['layerDependencies'] + | ||
167 | "?filter=layerbranch__branch__name:%s" % | ||
168 | "OR".join(whitelist_branch_names)) | ||
169 | |||
170 | dependlist = {} | ||
171 | for ldi in layerdependencies_info: | ||
172 | try: | ||
173 | lv = Layer_Version.objects.get( | ||
174 | pk=li_layer_branch_id_to_toaster_lv_id[ldi['layerbranch']]) | ||
175 | except Layer_Version.DoesNotExist as e: | ||
176 | continue | ||
177 | |||
178 | if lv not in dependlist: | ||
179 | dependlist[lv] = [] | ||
180 | try: | ||
181 | layer_id = li_layer_id_to_toaster_layer_id[ldi['dependency']] | ||
182 | |||
183 | dependlist[lv].append( | ||
184 | Layer_Version.objects.get( | ||
185 | layer_source=LayerSource.TYPE_LAYERINDEX, | ||
186 | layer__pk=layer_id)) | ||
187 | |||
188 | except Layer_Version.DoesNotExist: | ||
189 | logger.warning("Cannot find layer version (ls:%s)," | ||
190 | "up_id:%s lv:%s" % | ||
191 | (self, ldi['dependency'], lv)) | ||
192 | |||
193 | for lv in dependlist: | ||
194 | LayerVersionDependency.objects.filter(layer_version=lv).delete() | ||
195 | for lvd in dependlist[lv]: | ||
196 | LayerVersionDependency.objects.get_or_create(layer_version=lv, | ||
197 | depends_on=lvd) | ||
198 | |||
199 | # update machines | ||
200 | logger.debug("Fetching machine information") | ||
201 | machines_info = _get_json_response( | ||
202 | apilinks['machines'] + "?filter=layerbranch__branch__name:%s" % | ||
203 | "OR".join(whitelist_branch_names)) | ||
204 | |||
205 | for mi in machines_info: | ||
206 | mo, created = Machine.objects.get_or_create( | ||
207 | name=mi['name'], | ||
208 | layer_version=Layer_Version.objects.get( | ||
209 | pk=li_layer_branch_id_to_toaster_lv_id[mi['layerbranch']])) | ||
210 | mo.up_date = mi['updated'] | ||
211 | mo.name = mi['name'] | ||
212 | mo.description = mi['description'] | ||
213 | mo.save() | ||
214 | |||
215 | # update recipes; paginate by layer version / layer branch | ||
216 | logger.debug("Fetching target information") | ||
217 | recipes_info = _get_json_response( | ||
218 | apilinks['recipes'] + "?filter=layerbranch__branch__name:%s" % | ||
219 | "OR".join(whitelist_branch_names)) | ||
220 | |||
221 | for ri in recipes_info: | ||
222 | try: | ||
223 | lv_id = li_layer_branch_id_to_toaster_lv_id[ri['layerbranch']] | ||
224 | lv = Layer_Version.objects.get(pk=lv_id) | ||
225 | |||
226 | ro, created = Recipe.objects.get_or_create( | ||
227 | layer_version=lv, | ||
228 | name=ri['pn'] | ||
229 | ) | ||
8 | 230 | ||
231 | ro.layer_version = lv | ||
232 | ro.up_date = ri['updated'] | ||
233 | ro.name = ri['pn'] | ||
234 | ro.version = ri['pv'] | ||
235 | ro.summary = ri['summary'] | ||
236 | ro.description = ri['description'] | ||
237 | ro.section = ri['section'] | ||
238 | ro.license = ri['license'] | ||
239 | ro.homepage = ri['homepage'] | ||
240 | ro.bugtracker = ri['bugtracker'] | ||
241 | ro.file_path = ri['filepath'] + "/" + ri['filename'] | ||
242 | if 'inherits' in ri: | ||
243 | ro.is_image = 'image' in ri['inherits'].split() | ||
244 | else: # workaround for old style layer index | ||
245 | ro.is_image = "-image-" in ri['pn'] | ||
246 | ro.save() | ||
247 | except Exception as e: | ||
248 | logger.debug("Failed saving recipe %s", e) | ||
9 | 249 | ||
10 | def handle_noargs(self, **options): | 250 | def handle_noargs(self, **options): |
11 | for ls in LayerSource.objects.all(): | 251 | self.update() |
12 | ls.update() | ||