summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/toaster/orm/management
diff options
context:
space:
mode:
authorMichael Wood <michael.g.wood@intel.com>2016-07-21 14:43:29 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-07-26 08:10:36 +0100
commit8b3146007f98ba6618662cc47ed1c1491ef8945a (patch)
tree40a31813eab822a7a50eb6e62546559d61ab87cc /bitbake/lib/toaster/orm/management
parentbb260e94a63d6d498c9c76b862bc836a90678495 (diff)
downloadpoky-8b3146007f98ba6618662cc47ed1c1491ef8945a.tar.gz
bitbake: toaster: lsupdates Add progress information and clean up logging
Adds basic progress % information and provides better description of what is happening. (Bitbake rev: 6393dbf97d450d2521c0bc9429da0987bb7720ec) Signed-off-by: Michael Wood <michael.g.wood@intel.com> Signed-off-by: Elliot Smith <elliot.smith@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/toaster/orm/management')
-rw-r--r--bitbake/lib/toaster/orm/management/commands/lsupdates.py81
1 files changed, 58 insertions, 23 deletions
diff --git a/bitbake/lib/toaster/orm/management/commands/lsupdates.py b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
index 89ee53e3dd..138b686eb5 100644
--- a/bitbake/lib/toaster/orm/management/commands/lsupdates.py
+++ b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
@@ -25,6 +25,8 @@ from orm.models import LayerSource, Layer, Release, Layer_Version
25from orm.models import LayerVersionDependency, Machine, Recipe 25from orm.models import LayerVersionDependency, Machine, Recipe
26 26
27import os 27import os
28import sys
29
28import json 30import json
29import logging 31import logging
30logger = logging.getLogger("toaster") 32logger = logging.getLogger("toaster")
@@ -36,6 +38,18 @@ class Command(NoArgsCommand):
36 args = "" 38 args = ""
37 help = "Updates locally cached information from a layerindex server" 39 help = "Updates locally cached information from a layerindex server"
38 40
41 def mini_progress(self, what, i, total):
42 i = i + 1
43 pec = (float(i)/float(total))*100
44
45 sys.stdout.write("\rUpdating %s %d%%" %
46 (what,
47 pec))
48 sys.stdout.flush()
49 if int(pec) is 100:
50 sys.stdout.write("\n")
51 sys.stdout.flush()
52
39 def update(self): 53 def update(self):
40 """ 54 """
41 Fetches layer, recipe and machine information from a layerindex 55 Fetches layer, recipe and machine information from a layerindex
@@ -85,7 +99,8 @@ class Command(NoArgsCommand):
85 if len(whitelist_branch_names) == 0: 99 if len(whitelist_branch_names) == 0:
86 raise Exception("Failed to make list of branches to fetch") 100 raise Exception("Failed to make list of branches to fetch")
87 101
88 logger.debug("Fetching branches") 102 logger.info("Fetching metadata releases for %s",
103 " ".join(whitelist_branch_names))
89 104
90 # keep a track of the id mappings so that layer_versions can be created 105 # keep a track of the id mappings so that layer_versions can be created
91 # for these layers later on 106 # for these layers later on
@@ -96,10 +111,10 @@ class Command(NoArgsCommand):
96 # "?filter=name:%s" 111 # "?filter=name:%s"
97 # % "OR".join(whitelist_branch_names)) 112 # % "OR".join(whitelist_branch_names))
98 113
99 # update layers
100 layers_info = _get_json_response(apilinks['layerItems']) 114 layers_info = _get_json_response(apilinks['layerItems'])
101 115
102 for li in layers_info: 116 total = len(layers_info)
117 for i, li in enumerate(layers_info):
103 # Special case for the openembedded-core layer 118 # Special case for the openembedded-core layer
104 if li['name'] == oe_core_layer: 119 if li['name'] == oe_core_layer:
105 try: 120 try:
@@ -113,25 +128,34 @@ class Command(NoArgsCommand):
113 oe_core_l.description = li['description'] 128 oe_core_l.description = li['description']
114 oe_core_l.save() 129 oe_core_l.save()
115 li_layer_id_to_toaster_layer_id[li['id']] = oe_core_l.pk 130 li_layer_id_to_toaster_layer_id[li['id']] = oe_core_l.pk
131 self.mini_progress("layers", i, total)
116 continue 132 continue
117 133
118 except Layer.DoesNotExist: 134 except Layer.DoesNotExist:
119 pass 135 pass
120 136
121 l, created = Layer.objects.get_or_create(name=li['name']) 137 try:
122 l.up_date = li['updated'] 138 l, created = Layer.objects.get_or_create(name=li['name'],
123 l.vcs_url = li['vcs_url'] 139 vcs_url=li['vcs_url'])
124 l.vcs_web_url = li['vcs_web_url'] 140 l.up_date = li['updated']
125 l.vcs_web_tree_base_url = li['vcs_web_tree_base_url'] 141 l.vcs_url = li['vcs_url']
126 l.vcs_web_file_base_url = li['vcs_web_file_base_url'] 142 l.vcs_web_url = li['vcs_web_url']
127 l.summary = li['summary'] 143 l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
128 l.description = li['description'] 144 l.vcs_web_file_base_url = li['vcs_web_file_base_url']
129 l.save() 145 l.summary = li['summary']
146 l.description = li['description']
147 l.save()
148 except Layer.MultipleObjectsReturned:
149 logger.info("Skipped %s as we found multiple layers and "
150 "don't know which to update" %
151 li['name'])
130 152
131 li_layer_id_to_toaster_layer_id[li['id']] = l.pk 153 li_layer_id_to_toaster_layer_id[li['id']] = l.pk
132 154
133 # update layerbranches/layer_versions 155 self.mini_progress("layers", i, total)
134 logger.debug("Fetching layer information") 156
157 # update layer_versions
158 logger.info("Fetching layer versions")
135 layerbranches_info = _get_json_response( 159 layerbranches_info = _get_json_response(
136 apilinks['layerBranches'] + "?filter=branch__name:%s" % 160 apilinks['layerBranches'] + "?filter=branch__name:%s" %
137 "OR".join(whitelist_branch_names)) 161 "OR".join(whitelist_branch_names))
@@ -140,7 +164,8 @@ class Command(NoArgsCommand):
140 # layer_version toaster object id 164 # layer_version toaster object id
141 li_layer_branch_id_to_toaster_lv_id = {} 165 li_layer_branch_id_to_toaster_lv_id = {}
142 166
143 for lbi in layerbranches_info: 167 total = len(layerbranches_info)
168 for i, lbi in enumerate(layerbranches_info):
144 169
145 try: 170 try:
146 lv, created = Layer_Version.objects.get_or_create( 171 lv, created = Layer_Version.objects.get_or_create(
@@ -149,8 +174,9 @@ class Command(NoArgsCommand):
149 pk=li_layer_id_to_toaster_layer_id[lbi['layer']]) 174 pk=li_layer_id_to_toaster_layer_id[lbi['layer']])
150 ) 175 )
151 except KeyError: 176 except KeyError:
152 print("No such layerindex layer referenced by layerbranch %d" % 177 logger.warning(
153 lbi['layer']) 178 "No such layerindex layer referenced by layerbranch %d" %
179 lbi['layer'])
154 continue 180 continue
155 181
156 lv.up_date = lbi['updated'] 182 lv.up_date = lbi['updated']
@@ -160,7 +186,9 @@ class Command(NoArgsCommand):
160 186
161 li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\ 187 li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\
162 lv.pk 188 lv.pk
189 self.mini_progress("layer versions", i, total)
163 190
191 logger.info("Fetching layer version dependencies")
164 # update layer dependencies 192 # update layer dependencies
165 layerdependencies_info = _get_json_response( 193 layerdependencies_info = _get_json_response(
166 apilinks['layerDependencies'] + 194 apilinks['layerDependencies'] +
@@ -190,19 +218,22 @@ class Command(NoArgsCommand):
190 "up_id:%s lv:%s" % 218 "up_id:%s lv:%s" %
191 (self, ldi['dependency'], lv)) 219 (self, ldi['dependency'], lv))
192 220
193 for lv in dependlist: 221 total = len(dependlist)
222 for i, lv in enumerate(dependlist):
194 LayerVersionDependency.objects.filter(layer_version=lv).delete() 223 LayerVersionDependency.objects.filter(layer_version=lv).delete()
195 for lvd in dependlist[lv]: 224 for lvd in dependlist[lv]:
196 LayerVersionDependency.objects.get_or_create(layer_version=lv, 225 LayerVersionDependency.objects.get_or_create(layer_version=lv,
197 depends_on=lvd) 226 depends_on=lvd)
227 self.mini_progress("Layer version dependencies", i, total)
198 228
199 # update machines 229 # update machines
200 logger.debug("Fetching machine information") 230 logger.info("Fetching machine information")
201 machines_info = _get_json_response( 231 machines_info = _get_json_response(
202 apilinks['machines'] + "?filter=layerbranch__branch__name:%s" % 232 apilinks['machines'] + "?filter=layerbranch__branch__name:%s" %
203 "OR".join(whitelist_branch_names)) 233 "OR".join(whitelist_branch_names))
204 234
205 for mi in machines_info: 235 total = len(machines_info)
236 for i, mi in enumerate(machines_info):
206 mo, created = Machine.objects.get_or_create( 237 mo, created = Machine.objects.get_or_create(
207 name=mi['name'], 238 name=mi['name'],
208 layer_version=Layer_Version.objects.get( 239 layer_version=Layer_Version.objects.get(
@@ -211,14 +242,16 @@ class Command(NoArgsCommand):
211 mo.name = mi['name'] 242 mo.name = mi['name']
212 mo.description = mi['description'] 243 mo.description = mi['description']
213 mo.save() 244 mo.save()
245 self.mini_progress("machines", i, total)
214 246
215 # update recipes; paginate by layer version / layer branch 247 # update recipes; paginate by layer version / layer branch
216 logger.debug("Fetching target information") 248 logger.info("Fetching recipe information")
217 recipes_info = _get_json_response( 249 recipes_info = _get_json_response(
218 apilinks['recipes'] + "?filter=layerbranch__branch__name:%s" % 250 apilinks['recipes'] + "?filter=layerbranch__branch__name:%s" %
219 "OR".join(whitelist_branch_names)) 251 "OR".join(whitelist_branch_names))
220 252
221 for ri in recipes_info: 253 total = len(recipes_info)
254 for i, ri in enumerate(recipes_info):
222 try: 255 try:
223 lv_id = li_layer_branch_id_to_toaster_lv_id[ri['layerbranch']] 256 lv_id = li_layer_branch_id_to_toaster_lv_id[ri['layerbranch']]
224 lv = Layer_Version.objects.get(pk=lv_id) 257 lv = Layer_Version.objects.get(pk=lv_id)
@@ -245,7 +278,9 @@ class Command(NoArgsCommand):
245 ro.is_image = "-image-" in ri['pn'] 278 ro.is_image = "-image-" in ri['pn']
246 ro.save() 279 ro.save()
247 except Exception as e: 280 except Exception as e:
248 logger.debug("Failed saving recipe %s", e) 281 logger.warning("Failed saving recipe %s", e)
282
283 self.mini_progress("recipes", i, total)
249 284
250 def handle_noargs(self, **options): 285 def handle_noargs(self, **options):
251 self.update() 286 self.update()