summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/fetch2/wget.py
diff options
context:
space:
mode:
authorAníbal Limón <anibal.limon@linux.intel.com>2015-02-13 15:58:11 -0600
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-02-16 09:13:06 +0000
commit22cebb87b72ec6641f704d291d096ef63fa108a2 (patch)
tree092d5a434dfab43c99cf7989295d5f7c4308df99 /bitbake/lib/bb/fetch2/wget.py
parent62e171b72180bc9aab0e5ffb70e2aa2919c0d132 (diff)
downloadpoky-22cebb87b72ec6641f704d291d096ef63fa108a2.tar.gz
bitbake: fetch2: wget latest_versionstring improvments in get version by dir
Add support for scan every version directory using _check_latest_version makes code more robust because sometimes upstream projects publish new directories without files, causing don't find version. To support this new behaviour remove _check_latest_dir and replace for _check_latest_version_by_dir, (Bitbake rev: 1a75b3707743c32eec9d2cf566fb6bbea9f73784) Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/fetch2/wget.py')
-rw-r--r--bitbake/lib/bb/fetch2/wget.py108
1 files changed, 37 insertions, 71 deletions
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index c8b120cafd..809e9fc7c6 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -113,7 +113,6 @@ class Wget(FetchMethod):
113 """ 113 """
114 Find and group name, version and archive type in the given string s 114 Find and group name, version and archive type in the given string s
115 """ 115 """
116 bb.debug(3, "_parse_path(%s, %s)" % (regex.pattern, s))
117 116
118 m = regex.search(s) 117 m = regex.search(s)
119 if m: 118 if m:
@@ -199,44 +198,6 @@ class Wget(FetchMethod):
199 f.close() 198 f.close()
200 return fetchresult 199 return fetchresult
201 200
202 def _check_latest_dir(self, url, versionstring, ud, d):
203 """
204 Return the name of the directory with the greatest package version
205 If error or no version, return None
206 """
207 bb.debug(3, "DirURL: %s, %s" % (url, versionstring))
208 soup = BeautifulSoup(self._fetch_index(url, ud, d))
209 if not soup:
210 return None
211
212 valid = 0
213 prefix = ''
214 regex = re.compile("(\D*)((\d+[\.\-_])+(\d+))")
215 m = regex.search(versionstring)
216 if m:
217 version = ('', m.group(2), '')
218 prefix = m.group(1)
219 bb.debug(3, "version: %s, prefix: %s" % (version, prefix))
220 else:
221 version = ('', versionstring, '')
222
223 for href in soup.find_all('a', href=True):
224 bb.debug(3, "href: %s" % (href['href']))
225 if href['href'].find(versionstring) >= 0:
226 valid = 1
227 m = regex.search(href['href'].strip("/"))
228 if m:
229 thisversion = ('', m.group(2), '')
230 if thisversion and self._vercmp(version, thisversion) < 0:
231 version = thisversion
232
233 if valid:
234 bb.debug(3, "Would return %s" % (prefix+version[1]))
235 return prefix+version[1]
236 else:
237 bb.debug(3, "Not Valid")
238 return None
239
240 def _check_latest_version(self, url, package, package_regex, current_version, ud, d): 201 def _check_latest_version(self, url, package, package_regex, current_version, ud, d):
241 """ 202 """
242 Return the latest version of a package inside a given directory path 203 Return the latest version of a package inside a given directory path
@@ -252,8 +213,8 @@ class Wget(FetchMethod):
252 return "" 213 return ""
253 214
254 for line in soup.find_all('a', href=True): 215 for line in soup.find_all('a', href=True):
255 bb.debug(3, "line = '%s'" % (str(line)))
256 bb.debug(3, "line['href'] = '%s'" % (line['href'])) 216 bb.debug(3, "line['href'] = '%s'" % (line['href']))
217 bb.debug(3, "line = '%s'" % (str(line)))
257 218
258 newver = self._parse_path(package_regex, line['href']) 219 newver = self._parse_path(package_regex, line['href'])
259 if not newver: 220 if not newver:
@@ -267,12 +228,13 @@ class Wget(FetchMethod):
267 elif self._vercmp(version, newver) < 0: 228 elif self._vercmp(version, newver) < 0:
268 version = newver 229 version = newver
269 230
270 # check whether a valid package and version were found 231 pupver = re.sub('_', '.', version[1])
232
271 bb.debug(3, "*** %s -> UpstreamVersion = %s (CurrentVersion = %s)" % 233 bb.debug(3, "*** %s -> UpstreamVersion = %s (CurrentVersion = %s)" %
272 (package, version[1] or "N/A", current_version[1])) 234 (package, pupver or "N/A", current_version[1]))
273 235
274 if valid: 236 if valid:
275 return re.sub('_', '.', version[1]) 237 return pupver
276 238
277 return "" 239 return ""
278 240
@@ -353,14 +315,10 @@ class Wget(FetchMethod):
353 psuffix_regex = "(tar\.gz|tgz|tar\.bz2|zip|xz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)" 315 psuffix_regex = "(tar\.gz|tgz|tar\.bz2|zip|xz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)"
354 316
355 # match name, version and archive type of a package 317 # match name, version and archive type of a package
356 package_regex_comp = re.compile("(?P<name>%s?)\.?v?(?P<pver>%s)(?P<arch>%s)?[\.\-](?P<type>%s$)" 318 package_regex_comp = re.compile("(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.\-](?P<type>%s$)"
357 % (pn_regex, pver_regex, parch_regex, psuffix_regex)) 319 % (pn_regex, pver_regex, parch_regex, psuffix_regex))
358 self.suffix_regex_comp = re.compile(psuffix_regex) 320 self.suffix_regex_comp = re.compile(psuffix_regex)
359 321
360 # search for version matches on folders inside the path, like:
361 # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz
362 self.dirver_regex_comp = re.compile("(?P<dirver>[^/]*(\d+\.)*\d+([\-_]r\d+)*)/")
363
364 # compile regex, can be specific by package or generic regex 322 # compile regex, can be specific by package or generic regex
365 pn_regex = d.getVar('REGEX', True) 323 pn_regex = d.getVar('REGEX', True)
366 if pn_regex: 324 if pn_regex:
@@ -369,10 +327,10 @@ class Wget(FetchMethod):
369 version = self._parse_path(package_regex_comp, package) 327 version = self._parse_path(package_regex_comp, package)
370 if version: 328 if version:
371 package_custom_regex_comp = re.compile( 329 package_custom_regex_comp = re.compile(
372 "(?P<name>%s)(?P<pver>%s)(?P<arch>%s)?[\.\-](?P<type>%s)$" % 330 "(?P<name>%s)(?P<pver>%s)(?P<arch>%s)?[\.\-](?P<type>%s)" %
373 (re.escape(version[0]), pver_regex, parch_regex, psuffix_regex)) 331 (re.escape(version[0]), pver_regex, parch_regex, psuffix_regex))
374 else: 332 else:
375 package_custom_regex_comp = package_regex_comp 333 package_custom_regex_comp = None
376 334
377 return package_custom_regex_comp 335 return package_custom_regex_comp
378 336
@@ -383,33 +341,41 @@ class Wget(FetchMethod):
383 sanity check to ensure same name and type. 341 sanity check to ensure same name and type.
384 """ 342 """
385 package = ud.path.split("/")[-1] 343 package = ud.path.split("/")[-1]
386 regex_uri = d.getVar("REGEX_URI", True) 344 current_version = ['', d.getVar('PV', True), '']
387 newpath = regex_uri or ud.path
388 pupver = ""
389
390 package_regex = self._init_regexes(package, ud, d)
391
392 current_version = ('', d.getVar('PV', True), '')
393 345
394 """possible to have no version in pkg name, such as spectrum-fw""" 346 """possible to have no version in pkg name, such as spectrum-fw"""
395 if not re.search("\d+", package): 347 if not re.search("\d+", package):
396 return re.sub('_', '.', current_version[1]) 348 current_version[1] = re.sub('_', '.', current_version[1])
349 current_version[1] = re.sub('-', '.', current_version[1])
350 return current_version[1]
397 351
352 package_regex = self._init_regexes(package, ud, d)
353 if package_regex is None:
354 bb.warn("latest_versionstring: package %s don't match pattern" % (package))
355 return ""
356 bb.debug(3, "latest_versionstring, regex: %s" % (package_regex.pattern))
357
358 uri = ""
359 regex_uri = d.getVar("REGEX_URI", True)
398 if not regex_uri: 360 if not regex_uri:
399 # generate the new uri with the appropriate latest directory 361 path = ud.path.split(package)[0]
400 m = self.dirver_regex_comp.search(ud.path) 362
363 # search for version matches on folders inside the path, like:
364 # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz
365 dirver_regex = re.compile("(?P<dirver>[^/]*(\d+\.)*\d+([\-_]r\d+)*)/")
366 m = dirver_regex.search(path)
401 if m: 367 if m:
368 pn = d.getVar('PN', True)
402 dirver = m.group('dirver') 369 dirver = m.group('dirver')
403 newuri = bb.fetch.encodeurl([ud.type, ud.host, 370
404 ud.path.split(dirver)[0], ud.user, ud.pswd, {}]) 371 dirver_pn_regex = re.compile("%s\d?" % (re.escape(pn)))
405 new_dirver = self._check_latest_dir(newuri, dirver, ud, d) 372 if not dirver_pn_regex.search(dirver):
406 if new_dirver and dirver != new_dirver: 373 return self._check_latest_version_by_dir(dirver,
407 newpath = ud.path.replace(dirver, new_dirver, True) 374 package, package_regex, current_version, ud, d)
408 375
409 newpath = newpath.split(package)[0] or "/" # path to directory 376 uri = bb.fetch.encodeurl([ud.type, ud.host, path, ud.user, ud.pswd, {}])
410 newuri = bb.fetch.encodeurl([ud.type, ud.host, newpath, ud.user, ud.pswd, {}])
411 else: 377 else:
412 newuri = newpath 378 uri = regex_uri
413 379
414 return self._check_latest_version(newuri, package, package_regex, 380 return self._check_latest_version(uri, package, package_regex,
415 current_version, ud, d) 381 current_version, ud, d)