diff options
author | Adrian Dudau <adrian.dudau@enea.com> | 2013-12-12 13:38:32 +0100 |
---|---|---|
committer | Adrian Dudau <adrian.dudau@enea.com> | 2013-12-12 13:50:20 +0100 |
commit | e2e6f6fe07049f33cb6348780fa975162752e421 (patch) | |
tree | b1813295411235d1297a0ed642b1346b24fdfb12 /meta/lib/oe/distro_check.py | |
download | poky-e2e6f6fe07049f33cb6348780fa975162752e421.tar.gz |
initial commit of Enea Linux 3.1
Migrated from the internal git server on the dora-enea branch
Signed-off-by: Adrian Dudau <adrian.dudau@enea.com>
Diffstat (limited to 'meta/lib/oe/distro_check.py')
-rw-r--r-- | meta/lib/oe/distro_check.py | 383 |
1 files changed, 383 insertions, 0 deletions
diff --git a/meta/lib/oe/distro_check.py b/meta/lib/oe/distro_check.py new file mode 100644 index 0000000000..8ed5b0ec80 --- /dev/null +++ b/meta/lib/oe/distro_check.py | |||
@@ -0,0 +1,383 @@ | |||
1 | def get_links_from_url(url): | ||
2 | "Return all the href links found on the web location" | ||
3 | |||
4 | import urllib, sgmllib | ||
5 | |||
6 | class LinksParser(sgmllib.SGMLParser): | ||
7 | def parse(self, s): | ||
8 | "Parse the given string 's'." | ||
9 | self.feed(s) | ||
10 | self.close() | ||
11 | |||
12 | def __init__(self, verbose=0): | ||
13 | "Initialise an object passing 'verbose' to the superclass." | ||
14 | sgmllib.SGMLParser.__init__(self, verbose) | ||
15 | self.hyperlinks = [] | ||
16 | |||
17 | def start_a(self, attributes): | ||
18 | "Process a hyperlink and its 'attributes'." | ||
19 | for name, value in attributes: | ||
20 | if name == "href": | ||
21 | self.hyperlinks.append(value.strip('/')) | ||
22 | |||
23 | def get_hyperlinks(self): | ||
24 | "Return the list of hyperlinks." | ||
25 | return self.hyperlinks | ||
26 | |||
27 | sock = urllib.urlopen(url) | ||
28 | webpage = sock.read() | ||
29 | sock.close() | ||
30 | |||
31 | linksparser = LinksParser() | ||
32 | linksparser.parse(webpage) | ||
33 | return linksparser.get_hyperlinks() | ||
34 | |||
35 | def find_latest_numeric_release(url): | ||
36 | "Find the latest listed numeric release on the given url" | ||
37 | max=0 | ||
38 | maxstr="" | ||
39 | for link in get_links_from_url(url): | ||
40 | try: | ||
41 | release = float(link) | ||
42 | except: | ||
43 | release = 0 | ||
44 | if release > max: | ||
45 | max = release | ||
46 | maxstr = link | ||
47 | return maxstr | ||
48 | |||
49 | def is_src_rpm(name): | ||
50 | "Check if the link is pointing to a src.rpm file" | ||
51 | if name[-8:] == ".src.rpm": | ||
52 | return True | ||
53 | else: | ||
54 | return False | ||
55 | |||
56 | def package_name_from_srpm(srpm): | ||
57 | "Strip out the package name from the src.rpm filename" | ||
58 | strings = srpm.split('-') | ||
59 | package_name = strings[0] | ||
60 | for i in range(1, len (strings) - 1): | ||
61 | str = strings[i] | ||
62 | if not str[0].isdigit(): | ||
63 | package_name += '-' + str | ||
64 | return package_name | ||
65 | |||
66 | def clean_package_list(package_list): | ||
67 | "Removes multiple entries of packages and sorts the list" | ||
68 | set = {} | ||
69 | map(set.__setitem__, package_list, []) | ||
70 | return set.keys() | ||
71 | |||
72 | |||
73 | def get_latest_released_meego_source_package_list(): | ||
74 | "Returns list of all the name os packages in the latest meego distro" | ||
75 | |||
76 | package_names = [] | ||
77 | try: | ||
78 | f = open("/tmp/Meego-1.1", "r") | ||
79 | for line in f: | ||
80 | package_names.append(line[:-1] + ":" + "main") # Also strip the '\n' at the end | ||
81 | except IOError: pass | ||
82 | package_list=clean_package_list(package_names) | ||
83 | return "1.0", package_list | ||
84 | |||
85 | def get_source_package_list_from_url(url, section): | ||
86 | "Return a sectioned list of package names from a URL list" | ||
87 | |||
88 | bb.note("Reading %s: %s" % (url, section)) | ||
89 | links = get_links_from_url(url) | ||
90 | srpms = filter(is_src_rpm, links) | ||
91 | names_list = map(package_name_from_srpm, srpms) | ||
92 | |||
93 | new_pkgs = [] | ||
94 | for pkgs in names_list: | ||
95 | new_pkgs.append(pkgs + ":" + section) | ||
96 | |||
97 | return new_pkgs | ||
98 | |||
99 | def get_latest_released_fedora_source_package_list(): | ||
100 | "Returns list of all the name os packages in the latest fedora distro" | ||
101 | latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/") | ||
102 | |||
103 | package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main") | ||
104 | |||
105 | # package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Everything/source/SPRMS/" % latest, "everything") | ||
106 | package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates") | ||
107 | |||
108 | package_list=clean_package_list(package_names) | ||
109 | |||
110 | return latest, package_list | ||
111 | |||
112 | def get_latest_released_opensuse_source_package_list(): | ||
113 | "Returns list of all the name os packages in the latest opensuse distro" | ||
114 | latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/") | ||
115 | |||
116 | package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main") | ||
117 | package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates") | ||
118 | |||
119 | package_list=clean_package_list(package_names) | ||
120 | return latest, package_list | ||
121 | |||
122 | def get_latest_released_mandriva_source_package_list(): | ||
123 | "Returns list of all the name os packages in the latest mandriva distro" | ||
124 | latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/") | ||
125 | package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main") | ||
126 | # package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/contrib/release/" % latest, "contrib") | ||
127 | package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates") | ||
128 | |||
129 | package_list=clean_package_list(package_names) | ||
130 | return latest, package_list | ||
131 | |||
132 | def find_latest_debian_release(url): | ||
133 | "Find the latest listed debian release on the given url" | ||
134 | |||
135 | releases = [] | ||
136 | for link in get_links_from_url(url): | ||
137 | if link[:6] == "Debian": | ||
138 | if ';' not in link: | ||
139 | releases.append(link) | ||
140 | releases.sort() | ||
141 | try: | ||
142 | return releases.pop()[6:] | ||
143 | except: | ||
144 | return "_NotFound_" | ||
145 | |||
146 | def get_debian_style_source_package_list(url, section): | ||
147 | "Return the list of package-names stored in the debian style Sources.gz file" | ||
148 | import urllib | ||
149 | sock = urllib.urlopen(url) | ||
150 | import tempfile | ||
151 | tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False) | ||
152 | tmpfilename=tmpfile.name | ||
153 | tmpfile.write(sock.read()) | ||
154 | sock.close() | ||
155 | tmpfile.close() | ||
156 | import gzip | ||
157 | bb.note("Reading %s: %s" % (url, section)) | ||
158 | |||
159 | f = gzip.open(tmpfilename) | ||
160 | package_names = [] | ||
161 | for line in f: | ||
162 | if line[:9] == "Package: ": | ||
163 | package_names.append(line[9:-1] + ":" + section) # Also strip the '\n' at the end | ||
164 | os.unlink(tmpfilename) | ||
165 | |||
166 | return package_names | ||
167 | |||
168 | def get_latest_released_debian_source_package_list(): | ||
169 | "Returns list of all the name os packages in the latest debian distro" | ||
170 | latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/") | ||
171 | url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" | ||
172 | package_names = get_debian_style_source_package_list(url, "main") | ||
173 | # url = "http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.gz" | ||
174 | # package_names += get_debian_style_source_package_list(url, "contrib") | ||
175 | url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" | ||
176 | package_names += get_debian_style_source_package_list(url, "updates") | ||
177 | package_list=clean_package_list(package_names) | ||
178 | return latest, package_list | ||
179 | |||
180 | def find_latest_ubuntu_release(url): | ||
181 | "Find the latest listed ubuntu release on the given url" | ||
182 | url += "?C=M;O=D" # Descending Sort by Last Modified | ||
183 | for link in get_links_from_url(url): | ||
184 | if link[-8:] == "-updates": | ||
185 | return link[:-8] | ||
186 | return "_NotFound_" | ||
187 | |||
188 | def get_latest_released_ubuntu_source_package_list(): | ||
189 | "Returns list of all the name os packages in the latest ubuntu distro" | ||
190 | latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/") | ||
191 | url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest | ||
192 | package_names = get_debian_style_source_package_list(url, "main") | ||
193 | # url = "http://archive.ubuntu.com/ubuntu/dists/%s/multiverse/source/Sources.gz" % latest | ||
194 | # package_names += get_debian_style_source_package_list(url, "multiverse") | ||
195 | # url = "http://archive.ubuntu.com/ubuntu/dists/%s/universe/source/Sources.gz" % latest | ||
196 | # package_names += get_debian_style_source_package_list(url, "universe") | ||
197 | url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest | ||
198 | package_names += get_debian_style_source_package_list(url, "updates") | ||
199 | package_list=clean_package_list(package_names) | ||
200 | return latest, package_list | ||
201 | |||
202 | def create_distro_packages_list(distro_check_dir): | ||
203 | pkglst_dir = os.path.join(distro_check_dir, "package_lists") | ||
204 | if not os.path.isdir (pkglst_dir): | ||
205 | os.makedirs(pkglst_dir) | ||
206 | # first clear old stuff | ||
207 | for file in os.listdir(pkglst_dir): | ||
208 | os.unlink(os.path.join(pkglst_dir, file)) | ||
209 | |||
210 | per_distro_functions = [ | ||
211 | ["Debian", get_latest_released_debian_source_package_list], | ||
212 | ["Ubuntu", get_latest_released_ubuntu_source_package_list], | ||
213 | ["Fedora", get_latest_released_fedora_source_package_list], | ||
214 | ["OpenSuSE", get_latest_released_opensuse_source_package_list], | ||
215 | ["Mandriva", get_latest_released_mandriva_source_package_list], | ||
216 | ["Meego", get_latest_released_meego_source_package_list] | ||
217 | ] | ||
218 | |||
219 | from datetime import datetime | ||
220 | begin = datetime.now() | ||
221 | for distro in per_distro_functions: | ||
222 | name = distro[0] | ||
223 | release, package_list = distro[1]() | ||
224 | bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list))) | ||
225 | package_list_file = os.path.join(pkglst_dir, name + "-" + release) | ||
226 | f = open(package_list_file, "w+b") | ||
227 | for pkg in package_list: | ||
228 | f.write(pkg + "\n") | ||
229 | f.close() | ||
230 | end = datetime.now() | ||
231 | delta = end - begin | ||
232 | bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds) | ||
233 | |||
234 | def update_distro_data(distro_check_dir, datetime): | ||
235 | """ | ||
236 | If distro packages list data is old then rebuild it. | ||
237 | The operations has to be protected by a lock so that | ||
238 | only one thread performes it at a time. | ||
239 | """ | ||
240 | if not os.path.isdir (distro_check_dir): | ||
241 | try: | ||
242 | bb.note ("Making new directory: %s" % distro_check_dir) | ||
243 | os.makedirs (distro_check_dir) | ||
244 | except OSError: | ||
245 | raise Exception('Unable to create directory %s' % (distro_check_dir)) | ||
246 | |||
247 | |||
248 | datetime_file = os.path.join(distro_check_dir, "build_datetime") | ||
249 | saved_datetime = "_invalid_" | ||
250 | import fcntl | ||
251 | try: | ||
252 | if not os.path.exists(datetime_file): | ||
253 | open(datetime_file, 'w+b').close() # touch the file so that the next open won't fail | ||
254 | |||
255 | f = open(datetime_file, "r+b") | ||
256 | fcntl.lockf(f, fcntl.LOCK_EX) | ||
257 | saved_datetime = f.read() | ||
258 | if saved_datetime[0:8] != datetime[0:8]: | ||
259 | bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime)) | ||
260 | bb.note("Regenerating distro package lists") | ||
261 | create_distro_packages_list(distro_check_dir) | ||
262 | f.seek(0) | ||
263 | f.write(datetime) | ||
264 | |||
265 | except OSError: | ||
266 | raise Exception('Unable to read/write this file: %s' % (datetime_file)) | ||
267 | finally: | ||
268 | fcntl.lockf(f, fcntl.LOCK_UN) | ||
269 | f.close() | ||
270 | |||
271 | def compare_in_distro_packages_list(distro_check_dir, d): | ||
272 | if not os.path.isdir(distro_check_dir): | ||
273 | raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed") | ||
274 | |||
275 | localdata = bb.data.createCopy(d) | ||
276 | pkglst_dir = os.path.join(distro_check_dir, "package_lists") | ||
277 | matching_distros = [] | ||
278 | pn = d.getVar('PN', True) | ||
279 | recipe_name = d.getVar('PN', True) | ||
280 | bb.note("Checking: %s" % pn) | ||
281 | |||
282 | trim_dict = dict({"-native":"-native", "-cross":"-cross", "-initial":"-initial"}) | ||
283 | |||
284 | if pn.find("-native") != -1: | ||
285 | pnstripped = pn.split("-native") | ||
286 | localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) | ||
287 | bb.data.update_data(localdata) | ||
288 | recipe_name = pnstripped[0] | ||
289 | |||
290 | if pn.startswith("nativesdk-"): | ||
291 | pnstripped = pn.split("nativesdk-") | ||
292 | localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES', True)) | ||
293 | bb.data.update_data(localdata) | ||
294 | recipe_name = pnstripped[1] | ||
295 | |||
296 | if pn.find("-cross") != -1: | ||
297 | pnstripped = pn.split("-cross") | ||
298 | localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) | ||
299 | bb.data.update_data(localdata) | ||
300 | recipe_name = pnstripped[0] | ||
301 | |||
302 | if pn.find("-initial") != -1: | ||
303 | pnstripped = pn.split("-initial") | ||
304 | localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) | ||
305 | bb.data.update_data(localdata) | ||
306 | recipe_name = pnstripped[0] | ||
307 | |||
308 | bb.note("Recipe: %s" % recipe_name) | ||
309 | tmp = localdata.getVar('DISTRO_PN_ALIAS', True) | ||
310 | |||
311 | distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'}) | ||
312 | |||
313 | if tmp: | ||
314 | list = tmp.split(' ') | ||
315 | for str in list: | ||
316 | if str and str.find("=") == -1 and distro_exceptions[str]: | ||
317 | matching_distros.append(str) | ||
318 | |||
319 | distro_pn_aliases = {} | ||
320 | if tmp: | ||
321 | list = tmp.split(' ') | ||
322 | for str in list: | ||
323 | if str.find("=") != -1: | ||
324 | (dist, pn_alias) = str.split('=') | ||
325 | distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() | ||
326 | |||
327 | for file in os.listdir(pkglst_dir): | ||
328 | (distro, distro_release) = file.split("-") | ||
329 | f = open(os.path.join(pkglst_dir, file), "rb") | ||
330 | for line in f: | ||
331 | (pkg, section) = line.split(":") | ||
332 | if distro.lower() in distro_pn_aliases: | ||
333 | pn = distro_pn_aliases[distro.lower()] | ||
334 | else: | ||
335 | pn = recipe_name | ||
336 | if pn == pkg: | ||
337 | matching_distros.append(distro + "-" + section[:-1]) # strip the \n at the end | ||
338 | f.close() | ||
339 | break | ||
340 | f.close() | ||
341 | |||
342 | |||
343 | if tmp != None: | ||
344 | list = tmp.split(' ') | ||
345 | for item in list: | ||
346 | matching_distros.append(item) | ||
347 | bb.note("Matching: %s" % matching_distros) | ||
348 | return matching_distros | ||
349 | |||
350 | def create_log_file(d, logname): | ||
351 | import subprocess | ||
352 | logpath = d.getVar('LOG_DIR', True) | ||
353 | bb.utils.mkdirhier(logpath) | ||
354 | logfn, logsuffix = os.path.splitext(logname) | ||
355 | logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME', True), logsuffix)) | ||
356 | if not os.path.exists(logfile): | ||
357 | slogfile = os.path.join(logpath, logname) | ||
358 | if os.path.exists(slogfile): | ||
359 | os.remove(slogfile) | ||
360 | subprocess.call("touch %s" % logfile, shell=True) | ||
361 | os.symlink(logfile, slogfile) | ||
362 | d.setVar('LOG_FILE', logfile) | ||
363 | return logfile | ||
364 | |||
365 | |||
366 | def save_distro_check_result(result, datetime, result_file, d): | ||
367 | pn = d.getVar('PN', True) | ||
368 | logdir = d.getVar('LOG_DIR', True) | ||
369 | if not logdir: | ||
370 | bb.error("LOG_DIR variable is not defined, can't write the distro_check results") | ||
371 | return | ||
372 | if not os.path.isdir(logdir): | ||
373 | os.makedirs(logdir) | ||
374 | line = pn | ||
375 | for i in result: | ||
376 | line = line + "," + i | ||
377 | f = open(result_file, "a") | ||
378 | import fcntl | ||
379 | fcntl.lockf(f, fcntl.LOCK_EX) | ||
380 | f.seek(0, os.SEEK_END) # seek to the end of file | ||
381 | f.write(line + "\n") | ||
382 | fcntl.lockf(f, fcntl.LOCK_UN) | ||
383 | f.close() | ||