diff options
author | Tudor Florea <tudor.florea@enea.com> | 2015-10-09 22:59:03 +0200 |
---|---|---|
committer | Tudor Florea <tudor.florea@enea.com> | 2015-10-09 22:59:03 +0200 |
commit | 972dcfcdbfe75dcfeb777150c136576cf1a71e99 (patch) | |
tree | 97a61cd7e293d7ae9d56ef7ed0f81253365bb026 /bitbake/lib/toaster/bldcontrol/management | |
download | poky-972dcfcdbfe75dcfeb777150c136576cf1a71e99.tar.gz |
initial commit for Enea Linux 5.0 arm
Signed-off-by: Tudor Florea <tudor.florea@enea.com>
Diffstat (limited to 'bitbake/lib/toaster/bldcontrol/management')
4 files changed, 302 insertions, 0 deletions
diff --git a/bitbake/lib/toaster/bldcontrol/management/__init__.py b/bitbake/lib/toaster/bldcontrol/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/bitbake/lib/toaster/bldcontrol/management/__init__.py | |||
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py b/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py | |||
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py new file mode 100644 index 0000000000..4f6a66e711 --- /dev/null +++ b/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py | |||
@@ -0,0 +1,212 @@ | |||
1 | from django.core.management.base import NoArgsCommand, CommandError | ||
2 | from django.db import transaction | ||
3 | from orm.models import LayerSource, ToasterSetting, Branch, Layer, Layer_Version | ||
4 | from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer | ||
5 | from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException | ||
6 | from bldcontrol.models import BuildRequest, BuildEnvironment | ||
7 | import os | ||
8 | |||
9 | def DN(path): | ||
10 | if path is None: | ||
11 | return "" | ||
12 | else: | ||
13 | return os.path.dirname(path) | ||
14 | |||
15 | |||
16 | class Command(NoArgsCommand): | ||
17 | args = "" | ||
18 | help = "Verifies that the configured settings are valid and usable, or prompts the user to fix the settings." | ||
19 | |||
20 | def _reduce_canon_path(self, path): | ||
21 | components = [] | ||
22 | for c in path.split("/"): | ||
23 | if c == "..": | ||
24 | del components[-1] | ||
25 | elif c == ".": | ||
26 | pass | ||
27 | else: | ||
28 | components.append(c) | ||
29 | return "/".join(components) | ||
30 | |||
31 | def _find_first_path_for_file(self, startdirectory, filename, level = 0): | ||
32 | if level < 0: | ||
33 | return None | ||
34 | dirs = [] | ||
35 | for i in os.listdir(startdirectory): | ||
36 | j = os.path.join(startdirectory, i) | ||
37 | if os.path.isfile(j): | ||
38 | if i == filename: | ||
39 | return startdirectory | ||
40 | elif os.path.isdir(j): | ||
41 | dirs.append(j) | ||
42 | for j in dirs: | ||
43 | ret = self._find_first_path_for_file(j, filename, level - 1) | ||
44 | if ret is not None: | ||
45 | return ret | ||
46 | return None | ||
47 | |||
48 | def _get_suggested_sourcedir(self, be): | ||
49 | if be.betype != BuildEnvironment.TYPE_LOCAL: | ||
50 | return "" | ||
51 | return DN(DN(DN(self._find_first_path_for_file(self.guesspath, "toasterconf.json", 4)))) | ||
52 | |||
53 | def _get_suggested_builddir(self, be): | ||
54 | if be.betype != BuildEnvironment.TYPE_LOCAL: | ||
55 | return "" | ||
56 | return DN(self._find_first_path_for_file(self.guesspath, "bblayers.conf", 3)) | ||
57 | |||
58 | def _import_layer_config(self, baselayerdir): | ||
59 | filepath = os.path.join(baselayerdir, "meta/conf/toasterconf.json") | ||
60 | if not os.path.exists(filepath) or not os.path.isfile(filepath): | ||
61 | raise Exception("Failed to find toaster config file %s ." % filepath) | ||
62 | |||
63 | import json, pprint | ||
64 | data = json.loads(open(filepath, "r").read()) | ||
65 | |||
66 | # verify config file validity before updating settings | ||
67 | for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']: | ||
68 | assert i in data | ||
69 | |||
70 | # import bitbake data | ||
71 | for bvi in data['bitbake']: | ||
72 | bvo, created = BitbakeVersion.objects.get_or_create(name=bvi['name']) | ||
73 | bvo.giturl = bvi['giturl'] | ||
74 | bvo.branch = bvi['branch'] | ||
75 | bvo.dirpath = bvi['dirpath'] | ||
76 | bvo.save() | ||
77 | |||
78 | # set the layer sources | ||
79 | for lsi in data['layersources']: | ||
80 | assert 'sourcetype' in lsi | ||
81 | assert 'apiurl' in lsi | ||
82 | assert 'name' in lsi | ||
83 | assert 'branches' in lsi | ||
84 | |||
85 | if lsi['sourcetype'] == LayerSource.TYPE_LAYERINDEX or lsi['apiurl'].startswith("/"): | ||
86 | apiurl = lsi['apiurl'] | ||
87 | else: | ||
88 | apiurl = self._reduce_canon_path(os.path.join(DN(filepath), lsi['apiurl'])) | ||
89 | |||
90 | try: | ||
91 | ls = LayerSource.objects.get(sourcetype = lsi['sourcetype'], apiurl = apiurl) | ||
92 | except LayerSource.DoesNotExist: | ||
93 | ls = LayerSource.objects.create( | ||
94 | name = lsi['name'], | ||
95 | sourcetype = lsi['sourcetype'], | ||
96 | apiurl = apiurl | ||
97 | ) | ||
98 | |||
99 | layerbranches = [] | ||
100 | for branchname in lsi['branches']: | ||
101 | bo, created = Branch.objects.get_or_create(layer_source = ls, name = branchname) | ||
102 | layerbranches.append(bo) | ||
103 | |||
104 | if 'layers' in lsi: | ||
105 | for layerinfo in lsi['layers']: | ||
106 | lo, created = Layer.objects.get_or_create(layer_source = ls, name = layerinfo['name']) | ||
107 | if layerinfo['local_path'].startswith("/"): | ||
108 | lo.local_path = layerinfo['local_path'] | ||
109 | else: | ||
110 | lo.local_path = self._reduce_canon_path(os.path.join(DN(filepath), layerinfo['local_path'])) | ||
111 | lo.layer_index_url = layerinfo['layer_index_url'] | ||
112 | if 'vcs_url' in layerinfo: | ||
113 | lo.vcs_url = layerinfo['vcs_url'] | ||
114 | lo.save() | ||
115 | |||
116 | for branch in layerbranches: | ||
117 | lvo, created = Layer_Version.objects.get_or_create(layer_source = ls, | ||
118 | up_branch = branch, | ||
119 | commit = branch.name, | ||
120 | layer = lo) | ||
121 | lvo.dirpath = layerinfo['dirpath'] | ||
122 | lvo.save() | ||
123 | # set releases | ||
124 | for ri in data['releases']: | ||
125 | bvo = BitbakeVersion.objects.get(name = ri['bitbake']) | ||
126 | assert bvo is not None | ||
127 | |||
128 | ro, created = Release.objects.get_or_create(name = ri['name'], bitbake_version = bvo) | ||
129 | ro.description = ri['description'] | ||
130 | ro.branch = ri['branch'] | ||
131 | ro.save() | ||
132 | |||
133 | for dli in ri['defaultlayers']: | ||
134 | lsi, layername = dli.split(":") | ||
135 | layer, created = Layer.objects.get_or_create( | ||
136 | layer_source = LayerSource.objects.get(name = lsi), | ||
137 | name = layername | ||
138 | ) | ||
139 | ReleaseDefaultLayer.objects.get_or_create( release = ro, layer = layer) | ||
140 | |||
141 | # set default release | ||
142 | if ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").count() > 0: | ||
143 | ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").update(value = data['defaultrelease']) | ||
144 | else: | ||
145 | ToasterSetting.objects.create(name = "DEFAULT_RELEASE", value = data['defaultrelease']) | ||
146 | |||
147 | # set default config variables | ||
148 | for configname in data['config']: | ||
149 | if ToasterSetting.objects.filter(name = "DEFCONF_" + configname).count() > 0: | ||
150 | ToasterSetting.objects.filter(name = "DEFCONF_" + configname).update(value = data['config'][configname]) | ||
151 | else: | ||
152 | ToasterSetting.objects.create(name = "DEFCONF_" + configname, value = data['config'][configname]) | ||
153 | |||
154 | def handle(self, **options): | ||
155 | self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__))))))) | ||
156 | |||
157 | # we make sure we have builddir and sourcedir for all defined build envionments | ||
158 | for be in BuildEnvironment.objects.all(): | ||
159 | def _verify_be(): | ||
160 | is_changed = False | ||
161 | print("Verifying the Build Environment type %s id %d." % (be.get_betype_display(), be.pk)) | ||
162 | if len(be.sourcedir) == 0: | ||
163 | suggesteddir = self._get_suggested_sourcedir(be) | ||
164 | be.sourcedir = raw_input(" -- Layer sources checkout directory may not be empty [guessed \"%s\"]:" % suggesteddir) | ||
165 | if len(be.sourcedir) == 0 and len(suggesteddir) > 0: | ||
166 | be.sourcedir = suggesteddir | ||
167 | is_changed = True | ||
168 | |||
169 | if not be.sourcedir.startswith("/"): | ||
170 | be.sourcedir = raw_input(" -- Layer sources checkout directory must be an absolute path:") | ||
171 | is_changed = True | ||
172 | |||
173 | if len(be.builddir) == 0: | ||
174 | suggesteddir = self._get_suggested_builddir(be) | ||
175 | be.builddir = raw_input(" -- Build directory may not be empty [guessed \"%s\"]:" % suggesteddir) | ||
176 | if len(be.builddir) == 0 and len(suggesteddir) > 0: | ||
177 | be.builddir = suggesteddir | ||
178 | is_changed = True | ||
179 | |||
180 | if not be.builddir.startswith("/"): | ||
181 | be.builddir = raw_input(" -- Build directory must be an absolute path:") | ||
182 | is_changed = True | ||
183 | |||
184 | |||
185 | |||
186 | if is_changed: | ||
187 | print "Build configuration saved" | ||
188 | be.save() | ||
189 | |||
190 | if is_changed and be.betype == BuildEnvironment.TYPE_LOCAL: | ||
191 | baselayerdir = DN(DN(self._find_first_path_for_file(be.sourcedir, "toasterconf.json", 3))) | ||
192 | if baselayerdir: | ||
193 | i = raw_input(" -- Do you want to import basic layer configuration from \"%s\" ? (y/N):" % baselayerdir) | ||
194 | if len(i) and i.upper()[0] == 'Y': | ||
195 | self._import_layer_config(baselayerdir) | ||
196 | # we run lsupdates after config update | ||
197 | print "Updating information from the layer source, please wait." | ||
198 | from django.core.management import call_command | ||
199 | call_command("lsupdates") | ||
200 | pass | ||
201 | |||
202 | return is_changed | ||
203 | |||
204 | while (_verify_be()): | ||
205 | pass | ||
206 | |||
207 | # verify that default settings are there | ||
208 | if ToasterSetting.objects.filter(name = 'DEFAULT_RELEASE').count() != 1: | ||
209 | ToasterSetting.objects.filter(name = 'DEFAULT_RELEASE').delete() | ||
210 | ToasterSetting.objects.get_or_create(name = 'DEFAULT_RELEASE', value = '') | ||
211 | |||
212 | return 0 | ||
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py new file mode 100644 index 0000000000..8efe8e62bc --- /dev/null +++ b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py | |||
@@ -0,0 +1,90 @@ | |||
1 | from django.core.management.base import NoArgsCommand, CommandError | ||
2 | from django.db import transaction | ||
3 | from orm.models import Build | ||
4 | from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException | ||
5 | from bldcontrol.models import BuildRequest, BuildEnvironment, BRError | ||
6 | import os | ||
7 | |||
8 | class Command(NoArgsCommand): | ||
9 | args = "" | ||
10 | help = "Schedules and executes build requests as possible. Does not return (interrupt with Ctrl-C)" | ||
11 | |||
12 | |||
13 | @transaction.commit_on_success | ||
14 | def _selectBuildEnvironment(self): | ||
15 | bec = getBuildEnvironmentController(lock = BuildEnvironment.LOCK_FREE) | ||
16 | bec.be.lock = BuildEnvironment.LOCK_LOCK | ||
17 | bec.be.save() | ||
18 | return bec | ||
19 | |||
20 | @transaction.commit_on_success | ||
21 | def _selectBuildRequest(self): | ||
22 | br = BuildRequest.objects.filter(state = BuildRequest.REQ_QUEUED).order_by('pk')[0] | ||
23 | br.state = BuildRequest.REQ_INPROGRESS | ||
24 | br.save() | ||
25 | return br | ||
26 | |||
27 | def schedule(self): | ||
28 | import traceback | ||
29 | try: | ||
30 | br = None | ||
31 | try: | ||
32 | # select the build environment and the request to build | ||
33 | br = self._selectBuildRequest() | ||
34 | except IndexError as e: | ||
35 | return | ||
36 | try: | ||
37 | bec = self._selectBuildEnvironment() | ||
38 | except IndexError as e: | ||
39 | # we could not find a BEC; postpone the BR | ||
40 | br.state = BuildRequest.REQ_QUEUED | ||
41 | br.save() | ||
42 | return | ||
43 | |||
44 | # set up the buid environment with the needed layers | ||
45 | print "Build %s, Environment %s" % (br, bec.be) | ||
46 | bec.setLayers(br.brbitbake_set.all(), br.brlayer_set.all()) | ||
47 | |||
48 | # get the bb server running | ||
49 | bbctrl = bec.getBBController() | ||
50 | |||
51 | # let toasterui that this is a managed build | ||
52 | bbctrl.setVariable("TOASTER_BRBE", "%d:%d" % (br.pk, bec.be.pk)) | ||
53 | |||
54 | # set the build configuration | ||
55 | for variable in br.brvariable_set.all(): | ||
56 | bbctrl.setVariable(variable.name, variable.value) | ||
57 | |||
58 | # trigger the build command | ||
59 | bbctrl.build(list(map(lambda x:x.target, br.brtarget_set.all()))) | ||
60 | |||
61 | print "Build launched, exiting" | ||
62 | # disconnect from the server | ||
63 | bbctrl.disconnect() | ||
64 | |||
65 | # cleanup to be performed by toaster when the deed is done | ||
66 | |||
67 | |||
68 | except Exception as e: | ||
69 | print " EE Error executing shell command\n", e | ||
70 | traceback.print_exc(e) | ||
71 | BRError.objects.create(req = br, | ||
72 | errtype = str(type(e)), | ||
73 | errmsg = str(e), | ||
74 | traceback = traceback.format_exc(e)) | ||
75 | br.state = BuildRequest.REQ_FAILED | ||
76 | br.save() | ||
77 | bec.be.lock = BuildEnvironment.LOCK_FREE | ||
78 | bec.be.save() | ||
79 | |||
80 | |||
81 | def cleanup(self): | ||
82 | from django.utils import timezone | ||
83 | from datetime import timedelta | ||
84 | # environments locked for more than 30 seconds - they should be unlocked | ||
85 | BuildEnvironment.objects.filter(lock=BuildEnvironment.LOCK_LOCK).filter(updated__lt = timezone.now() - timedelta(seconds = 30)).update(lock = BuildEnvironment.LOCK_FREE) | ||
86 | |||
87 | |||
88 | def handle_noargs(self, **options): | ||
89 | self.cleanup() | ||
90 | self.schedule() | ||