summaryrefslogtreecommitdiffstats
path: root/meta
diff options
context:
space:
mode:
Diffstat (limited to 'meta')
-rw-r--r--meta/classes/utility-tasks.bbclass22
-rw-r--r--meta/lib/oe/distro_check.py298
2 files changed, 320 insertions, 0 deletions
diff --git a/meta/classes/utility-tasks.bbclass b/meta/classes/utility-tasks.bbclass
index 8bec8d463a..64d65c8aed 100644
--- a/meta/classes/utility-tasks.bbclass
+++ b/meta/classes/utility-tasks.bbclass
@@ -95,3 +95,25 @@ do_buildall() {
95 : 95 :
96} 96}
97 97
98addtask distro_check after do_distro_check
99do_distro_check[nostamp] = "1"
100python do_distro_check() {
101 """checks if the package is present in other public Linux distros"""
102 import oe.distro_check as dc
103 localdata = bb.data.createCopy(d)
104 bb.data.update_data(localdata)
105
106 tmpdir = bb.data.getVar('TMPDIR', localdata, 1)
107 distro_check_dir = os.path.join(tmpdir, "distro_check")
108 datetime = bb.data.getVar('DATETIME', localdata, 1)
109
110 # if distro packages list data is old then rebuild it
111 dc.update_distro_data(distro_check_dir, datetime)
112
113 # do the comparison
114 result = dc.compare_in_distro_packages_list(distro_check_dir, d)
115
116 # save the results
117 dc.save_distro_check_result(result, datetime, d)
118}
119
diff --git a/meta/lib/oe/distro_check.py b/meta/lib/oe/distro_check.py
new file mode 100644
index 0000000000..189f5ef20d
--- /dev/null
+++ b/meta/lib/oe/distro_check.py
@@ -0,0 +1,298 @@
1def get_links_from_url(url):
2 "Return all the href links found on the web location"
3
4 import urllib, sgmllib
5
6 class LinksParser(sgmllib.SGMLParser):
7 def parse(self, s):
8 "Parse the given string 's'."
9 self.feed(s)
10 self.close()
11
12 def __init__(self, verbose=0):
13 "Initialise an object passing 'verbose' to the superclass."
14 sgmllib.SGMLParser.__init__(self, verbose)
15 self.hyperlinks = []
16
17 def start_a(self, attributes):
18 "Process a hyperlink and its 'attributes'."
19 for name, value in attributes:
20 if name == "href":
21 self.hyperlinks.append(value.strip('/'))
22
23 def get_hyperlinks(self):
24 "Return the list of hyperlinks."
25 return self.hyperlinks
26
27 sock = urllib.urlopen(url)
28 webpage = sock.read()
29 sock.close()
30
31 linksparser = LinksParser()
32 linksparser.parse(webpage)
33 return linksparser.get_hyperlinks()
34
35def find_latest_numeric_release(url):
36 "Find the latest listed numeric release on the given url"
37 max=0
38 maxstr=""
39 for link in get_links_from_url(url):
40 try:
41 release = float(link)
42 except:
43 release = 0
44 if release > max:
45 max = release
46 maxstr = link
47 return maxstr
48
49def is_src_rpm(name):
50 "Check if the link is pointing to a src.rpm file"
51 if name[-8:] == ".src.rpm":
52 return True
53 else:
54 return False
55
56def package_name_from_srpm(srpm):
57 "Strip out the package name from the src.rpm filename"
58 strings = srpm.split('-')
59 package_name = strings[0]
60 for i in range(1, len (strings) - 1):
61 str = strings[i]
62 if not str[0].isdigit():
63 package_name += '-' + str
64 return package_name
65
66def clean_package_list(package_list):
67 "Removes multiple entries of packages and sorts the list"
68 set = {}
69 map(set.__setitem__, package_list, [])
70 return set.keys()
71
72
73def get_latest_released_fedora_source_package_list():
74 "Returns list of all the name os packages in the latest fedora distro"
75 latest = find_latest_numeric_release("http://download.fedora.redhat.com/pub/fedora/linux/releases/")
76
77 url = "http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest
78 links = get_links_from_url(url)
79 url = "http://download.fedora.redhat.com/pub/fedora/linux/updates/%s/SRPMS/" % latest
80 links += get_links_from_url(url)
81
82 srpms = filter(is_src_rpm, links)
83
84 package_names = map(package_name_from_srpm, srpms)
85 package_list=clean_package_list(package_names)
86
87 return latest, package_list
88
89def get_latest_released_opensuse_source_package_list():
90 "Returns list of all the name os packages in the latest opensuse distro"
91 latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/")
92
93 url = "http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest
94 links = get_links_from_url(url)
95 url = "http://download.opensuse.org/update/%s/rpm/src/" % latest
96 links += get_links_from_url(url)
97 srpms = filter(is_src_rpm, links)
98
99 package_names = map(package_name_from_srpm, srpms)
100 package_list=clean_package_list(package_names)
101 return latest, package_list
102
103def get_latest_released_mandriva_source_package_list():
104 "Returns list of all the name os packages in the latest mandriva distro"
105 latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/")
106 url = "http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest
107 links = get_links_from_url(url)
108 url = "http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest
109 links += get_links_from_url(url)
110
111 srpms = filter(is_src_rpm, links)
112
113 package_names = map(package_name_from_srpm, srpms)
114 package_list=clean_package_list(package_names)
115 return latest, package_list
116
117def find_latest_debian_release(url):
118 "Find the latest listed debian release on the given url"
119
120 releases = []
121 for link in get_links_from_url(url):
122 if link[:6] == "Debian":
123 if ';' not in link:
124 releases.append(link)
125 releases.sort()
126 try:
127 return releases.pop()[6:]
128 except:
129 return "_NotFound_"
130
131def get_debian_style_source_package_list(url):
132 "Return the list of package-names stored in the debian style Sources.gz file"
133 import urllib
134 sock = urllib.urlopen(url)
135 import tempfile
136 tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='poky.', suffix='.tmp', delete=False)
137 tmpfilename=tmpfile.name
138 tmpfile.write(sock.read())
139 sock.close()
140 tmpfile.close()
141 import gzip
142
143 f = gzip.open(tmpfilename)
144 package_names = []
145 for line in f:
146 if line[:9] == "Package: ":
147 package_names.append(line[9:-1]) # Also strip the '\n' at the end
148 os.unlink(tmpfilename)
149
150 return package_names
151
152def get_latest_released_debian_source_package_list():
153 "Returns list of all the name os packages in the latest debian distro"
154 latest = find_latest_debian_release("ftp://ftp.debian.org/debian/dists/")
155 url = "ftp://ftp.debian.org/debian/dists/stable/main/source/Sources.gz"
156 package_names = get_debian_style_source_package_list(url)
157 url = "ftp://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz"
158 package_names += get_debian_style_source_package_list(url)
159 package_list=clean_package_list(package_names)
160 return latest, package_list
161
162def find_latest_ubuntu_release(url):
163 "Find the latest listed ubuntu release on the given url"
164 url += "?C=M;O=D" # Descending Sort by Last Modified
165 for link in get_links_from_url(url):
166 if link[-8:] == "-updates":
167 return link[:-8]
168 return "_NotFound_"
169
170def get_latest_released_ubuntu_source_package_list():
171 "Returns list of all the name os packages in the latest ubuntu distro"
172 latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/")
173 url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest
174 package_names = get_debian_style_source_package_list(url)
175 url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest
176 package_names += get_debian_style_source_package_list(url)
177 package_list=clean_package_list(package_names)
178 return latest, package_list
179
180def create_distro_packages_list(distro_check_dir):
181 pkglst_dir = os.path.join(distro_check_dir, "package_lists")
182 if not os.path.isdir (pkglst_dir):
183 os.makedirs(pkglst_dir)
184 # first clear old stuff
185 for file in os.listdir(pkglst_dir):
186 os.unlink(os.path.join(pkglst_dir, file))
187
188 per_distro_functions = [["Fedora", get_latest_released_fedora_source_package_list],
189 ["OpenSuSE", get_latest_released_opensuse_source_package_list],
190 ["Ubuntu", get_latest_released_ubuntu_source_package_list],
191 ["Debian", get_latest_released_debian_source_package_list],
192 ["Mandriva", get_latest_released_mandriva_source_package_list]]
193
194 from datetime import datetime
195 begin = datetime.now()
196 for distro in per_distro_functions:
197 name = distro[0]
198 release, package_list = distro[1]()
199 bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list)))
200 package_list_file = os.path.join(pkglst_dir, name + "-" + release)
201 f = open(package_list_file, "w+b")
202 for pkg in package_list:
203 f.write(pkg + "\n")
204 f.close()
205 end = datetime.now()
206 delta = end - begin
207 bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds)
208
209def update_distro_data(distro_check_dir, datetime):
210 """
211 If distro packages list data is old then rebuild it.
212 The operations has to be protected by a lock so that
213 only one thread performes it at a time.
214 """
215 if not os.path.isdir (distro_check_dir):
216 try:
217 bb.note ("Making new directory: %s" % distro_check_dir)
218 os.makedirs (distro_check_dir)
219 except OSError:
220 raise Exception('Unable to create directory %s' % (distro_check_dir))
221
222
223 datetime_file = os.path.join(distro_check_dir, "build_datetime")
224 saved_datetime = "_invalid_"
225 import fcntl
226 try:
227 if not os.path.exists(datetime_file):
228 open(datetime_file, 'w+b').close() # touch the file so that the next open won't fail
229
230 f = open(datetime_file, "r+b")
231 fcntl.lockf(f, fcntl.LOCK_EX)
232 saved_datetime = f.read()
233 if saved_datetime != datetime:
234 bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime))
235 bb.note("Regenerating distro package lists")
236 create_distro_packages_list(distro_check_dir)
237 f.seek(0)
238 f.write(datetime)
239
240 except OSError:
241 raise Exception('Unable to read/write this file: %s' % (datetime_file))
242 finally:
243 fcntl.lockf(f, fcntl.LOCK_UN)
244 f.close()
245
246def compare_in_distro_packages_list(distro_check_dir, d):
247 if not os.path.isdir(distro_check_dir):
248 raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed")
249
250 pkglst_dir = os.path.join(distro_check_dir, "package_lists")
251 matching_distros = []
252 recipe_name = bb.data.getVar('PN', d, True)
253 tmp = bb.data.getVar('DISTRO_PN_ALIAS', d, True)
254 distro_pn_aliases = {}
255 if tmp:
256 list = tmp.split(';')
257 for str in list:
258 (dist, pn_alias) = str.split('=')
259 distro_pn_aliases[dist.strip().lower()] = pn_alias.strip()
260
261 for file in os.listdir(pkglst_dir):
262 (distro, distro_release) = file.split("-")
263 f = open(os.path.join(pkglst_dir, file), "rb")
264 for pkg in f:
265 if distro.lower() in distro_pn_aliases:
266 pn = distro_pn_aliases[distro.lower()]
267 else:
268 pn = recipe_name
269 if pn == pkg[:-1]: # strip the \n at the end
270 matching_distros.append(distro)
271 f.close()
272 break
273 f.close()
274 return matching_distros
275
276def save_distro_check_result(result, datetime, d):
277 pn = bb.data.getVar('PN', d, True)
278 logdir = bb.data.getVar('LOG_DIR', d, True)
279 if not logdir:
280 bb.error("LOG_DIR variable is not defined, can't write the distro_check results")
281 return
282 if not os.path.isdir(logdir):
283 os.makedirs(logdir)
284 result_file = os.path.join(logdir, "distro_check-" + datetime + ".results")
285 line = pn + " : "
286 for i in result:
287 line = line + i + ", "
288 if result:
289 line = line[:-2] # Take out the comma at the end of line
290 if not os.path.exists(result_file):
291 open(result_file, 'w+b').close() # touch the file so that the next open won't fail
292 f = open(result_file, "a+b")
293 import fcntl
294 fcntl.lockf(f, fcntl.LOCK_EX)
295 f.seek(0, os.SEEK_END) # seek to the end of file
296 f.write(line + "\n")
297 fcntl.lockf(f, fcntl.LOCK_UN)
298 f.close()