summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--bitbake/lib/bb/fetch2/__init__.py119
-rw-r--r--bitbake/lib/bb/fetch2/cvs.py4
-rw-r--r--bitbake/lib/bb/fetch2/git.py11
-rw-r--r--bitbake/lib/bb/fetch2/hg.py8
-rw-r--r--bitbake/lib/bb/fetch2/svk.py8
5 files changed, 67 insertions, 83 deletions
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 282713f40f..9b378a85e7 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -282,15 +282,6 @@ def subprocess_setup():
282 # SIGPIPE errors are known issues with gzip/bash 282 # SIGPIPE errors are known issues with gzip/bash
283 signal.signal(signal.SIGPIPE, signal.SIG_DFL) 283 signal.signal(signal.SIGPIPE, signal.SIG_DFL)
284 284
285def download_update(result, target):
286 if os.path.exists(target):
287 return
288 if not result or not os.path.exists(result):
289 return
290 if target != result:
291 os.symlink(result, target)
292 return
293
294def get_autorev(d): 285def get_autorev(d):
295 # only not cache src rev in autorev case 286 # only not cache src rev in autorev case
296 if bb.data.getVar('BB_SRCREV_POLICY', d, True) != "cache": 287 if bb.data.getVar('BB_SRCREV_POLICY', d, True) != "cache":
@@ -401,7 +392,7 @@ def check_network_access(d, info = ""):
401 else: 392 else:
402 logger.debug(1, "Fetcher accessed the network with the command %s" % info) 393 logger.debug(1, "Fetcher accessed the network with the command %s" % info)
403 394
404def try_mirrors(d, uri, mirrors, check = False, force = False): 395def try_mirrors(d, uri, mirrors, check = False):
405 """ 396 """
406 Try to use a mirrored version of the sources. 397 Try to use a mirrored version of the sources.
407 This method will be automatically called before the fetchers go. 398 This method will be automatically called before the fetchers go.
@@ -410,41 +401,31 @@ def try_mirrors(d, uri, mirrors, check = False, force = False):
410 uri is the original uri we're trying to download 401 uri is the original uri we're trying to download
411 mirrors is the list of mirrors we're going to try 402 mirrors is the list of mirrors we're going to try
412 """ 403 """
413 fpath = os.path.join(data.getVar("DL_DIR", d, True), os.path.basename(uri))
414 if not check and os.access(fpath, os.R_OK) and not force:
415 logger.debug(1, "%s already exists, skipping checkout.", fpath)
416 return fpath
417
418 ld = d.createCopy() 404 ld = d.createCopy()
419 for (find, replace) in mirrors: 405 for (find, replace) in mirrors:
420 newuri = uri_replace(uri, find, replace, ld) 406 newuri = uri_replace(uri, find, replace, ld)
421 if newuri != uri: 407 if newuri == uri:
422 try: 408 continue
423 ud = FetchData(newuri, ld) 409 try:
424 except bb.fetch2.NoMethodError: 410 ud = FetchData(newuri, ld)
425 logger.debug(1, "No method for %s", uri)
426 continue
427
428 ud.setup_localpath(ld) 411 ud.setup_localpath(ld)
429 412
430 try: 413 if check:
431 if check: 414 found = ud.method.checkstatus(newuri, ud, ld)
432 found = ud.method.checkstatus(newuri, ud, ld) 415 if found:
433 if found: 416 return found
434 return found 417 else:
435 else: 418 if not ud.method.need_update(newuri, ud, ld):
436 ud.method.download(newuri, ud, ld)
437 if hasattr(ud.method,"build_mirror_data"):
438 ud.method.build_mirror_data(newuri, ud, ld)
439 return ud.localpath 419 return ud.localpath
440 except (bb.fetch2.MissingParameterError, 420 ud.method.download(newuri, ud, ld)
441 bb.fetch2.FetchError, 421 if hasattr(ud.method,"build_mirror_data"):
442 bb.fetch2.MD5SumError): 422 ud.method.build_mirror_data(newuri, ud, ld)
443 import sys 423 return ud.localpath
444 (type, value, traceback) = sys.exc_info() 424
445 logger.debug(2, "Mirror fetch failure: %s", value) 425 except bb.fetch2.BBFetchException:
446 bb.utils.remove(ud.localpath) 426 logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, uri))
447 continue 427 bb.utils.remove(ud.localpath)
428 continue
448 return None 429 return None
449 430
450def srcrev_internal_helper(ud, d, name): 431def srcrev_internal_helper(ud, d, name):
@@ -481,6 +462,7 @@ class FetchData(object):
481 A class which represents the fetcher state for a given URI. 462 A class which represents the fetcher state for a given URI.
482 """ 463 """
483 def __init__(self, url, d): 464 def __init__(self, url, d):
465 # localpath is the location of a downloaded result. If not set, the file is local.
484 self.localfile = "" 466 self.localfile = ""
485 self.localpath = None 467 self.localpath = None
486 self.lockfile = None 468 self.lockfile = None
@@ -594,11 +576,13 @@ class FetchMethod(object):
594 576
595 urls = property(getUrls, setUrls, None, "Urls property") 577 urls = property(getUrls, setUrls, None, "Urls property")
596 578
597 def forcefetch(self, url, urldata, d): 579 def need_update(self, url, ud, d):
598 """ 580 """
599 Force a fetch, even if localpath exists? 581 Force a fetch, even if localpath exists?
600 """ 582 """
601 return False 583 if os.path.exists(ud.localpath):
584 return False
585 return True
602 586
603 def supports_srcrev(self): 587 def supports_srcrev(self):
604 """ 588 """
@@ -694,12 +678,7 @@ class FetchMethod(object):
694 """ 678 """
695 Should premirrors be used? 679 Should premirrors be used?
696 """ 680 """
697 if urldata.method.forcefetch(url, urldata, d): 681 return True
698 return True
699 elif os.path.exists(urldata.donestamp) and os.path.exists(urldata.localfile):
700 return False
701 else:
702 return True
703 682
704 def checkstatus(self, url, urldata, d): 683 def checkstatus(self, url, urldata, d):
705 """ 684 """
@@ -842,36 +821,32 @@ class Fetch(object):
842 821
843 lf = bb.utils.lockfile(ud.lockfile) 822 lf = bb.utils.lockfile(ud.lockfile)
844 823
845 if m.try_premirror(u, ud, self.d): 824 if not m.need_update(u, ud, self.d):
846 # First try fetching uri, u, from PREMIRRORS 825 localpath = ud.localpath
826 elif m.try_premirror(u, ud, self.d):
847 mirrors = mirror_from_string(bb.data.getVar('PREMIRRORS', self.d, True)) 827 mirrors = mirror_from_string(bb.data.getVar('PREMIRRORS', self.d, True))
848 localpath = try_mirrors(self.d, u, mirrors, False, m.forcefetch(u, ud, self.d)) 828 localpath = try_mirrors(self.d, u, mirrors, False)
849 elif os.path.exists(ud.localfile): 829
850 localpath = ud.localfile 830 if bb.data.getVar("BB_FETCH_PREMIRRORONLY", self.d, True) is None:
851 831 if not localpath and m.need_update(u, ud, self.d):
852 download_update(localpath, ud.localpath) 832 try:
853 833 m.download(u, ud, self.d)
854 # Need to re-test forcefetch() which will return true if our copy is too old 834 if hasattr(m, "build_mirror_data"):
855 if m.forcefetch(u, ud, self.d) or not localpath: 835 m.build_mirror_data(u, ud, self.d)
856 # Next try fetching from the original uri, u 836 localpath = ud.localpath
857 try: 837
858 m.download(u, ud, self.d) 838 except BBFetchException:
859 if hasattr(m, "build_mirror_data"): 839 # Remove any incomplete file
860 m.build_mirror_data(u, ud, self.d) 840 bb.utils.remove(ud.localpath)
861 localpath = ud.localpath 841 mirrors = mirror_from_string(bb.data.getVar('MIRRORS', self.d, True))
862 download_update(localpath, ud.localpath) 842 localpath = try_mirrors (self.d, u, mirrors)
863
864 except FetchError:
865 # Remove any incomplete file
866 bb.utils.remove(ud.localpath)
867 # Finally, try fetching uri, u, from MIRRORS
868 mirrors = mirror_from_string(bb.data.getVar('MIRRORS', self.d, True))
869 localpath = try_mirrors (self.d, u, mirrors)
870 843
871 if not localpath or not os.path.exists(localpath): 844 if not localpath or not os.path.exists(localpath):
872 raise FetchError("Unable to fetch URL %s from any source." % u, u) 845 raise FetchError("Unable to fetch URL %s from any source." % u, u)
873 846
874 download_update(localpath, ud.localpath) 847 # The local fetcher can return an alternate path so we symlink
848 if os.path.exists(localpath) and not os.path.exists(ud.localpath):
849 os.symlink(localpath, ud.localpath)
875 850
876 if os.path.exists(ud.donestamp): 851 if os.path.exists(ud.donestamp):
877 # Touch the done stamp file to show active use of the download 852 # Touch the done stamp file to show active use of the download
diff --git a/bitbake/lib/bb/fetch2/cvs.py b/bitbake/lib/bb/fetch2/cvs.py
index b77e742c33..69c31e7561 100644
--- a/bitbake/lib/bb/fetch2/cvs.py
+++ b/bitbake/lib/bb/fetch2/cvs.py
@@ -65,9 +65,11 @@ class Cvs(FetchMethod):
65 65
66 ud.localfile = data.expand('%s_%s_%s_%s%s%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date, norecurse, fullpath), d) 66 ud.localfile = data.expand('%s_%s_%s_%s%s%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date, norecurse, fullpath), d)
67 67
68 def forcefetch(self, url, ud, d): 68 def need_update(self, url, ud, d):
69 if (ud.date == "now"): 69 if (ud.date == "now"):
70 return True 70 return True
71 if not os.path.exists(ud.localpath):
72 return True
71 return False 73 return False
72 74
73 def download(self, loc, ud, d): 75 def download(self, loc, ud, d):
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 55c66cf49a..6bcc4a4838 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -80,7 +80,7 @@ class Git(FetchMethod):
80 def localpath(self, url, ud, d): 80 def localpath(self, url, ud, d):
81 return ud.clonedir 81 return ud.clonedir
82 82
83 def forcefetch(self, url, ud, d): 83 def need_update(self, u, ud, d):
84 if not os.path.exists(ud.clonedir): 84 if not os.path.exists(ud.clonedir):
85 return True 85 return True
86 os.chdir(ud.clonedir) 86 os.chdir(ud.clonedir)
@@ -90,13 +90,12 @@ class Git(FetchMethod):
90 return False 90 return False
91 91
92 def try_premirror(self, u, ud, d): 92 def try_premirror(self, u, ud, d):
93 if 'noclone' in ud.parm: 93 # If we don't do this, updating an existing checkout with only premirrors
94 return False 94 # is not possible
95 if bb.data.getVar("BB_FETCH_PREMIRRORONLY", d, True) is not None:
96 return True
95 if os.path.exists(ud.clonedir): 97 if os.path.exists(ud.clonedir):
96 return False 98 return False
97 if os.path.exists(ud.localpath):
98 return False
99
100 return True 99 return True
101 100
102 def download(self, loc, ud, d): 101 def download(self, loc, ud, d):
diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py
index d186b009dd..ac5825baa9 100644
--- a/bitbake/lib/bb/fetch2/hg.py
+++ b/bitbake/lib/bb/fetch2/hg.py
@@ -64,9 +64,13 @@ class Hg(FetchMethod):
64 64
65 ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d) 65 ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d)
66 66
67 def forcefetch(self, url, ud, d): 67 def need_update(self, url, ud, d):
68 revTag = ud.parm.get('rev', 'tip') 68 revTag = ud.parm.get('rev', 'tip')
69 return revTag == "tip" 69 if revTag == "tip":
70 return True
71 if not os.path.exists(ud.localpath):
72 return True
73 return False
70 74
71 def _buildhgcommand(self, ud, d, command): 75 def _buildhgcommand(self, ud, d, command):
72 """ 76 """
diff --git a/bitbake/lib/bb/fetch2/svk.py b/bitbake/lib/bb/fetch2/svk.py
index 70f72c80ab..117a22f438 100644
--- a/bitbake/lib/bb/fetch2/svk.py
+++ b/bitbake/lib/bb/fetch2/svk.py
@@ -53,8 +53,12 @@ class Svk(FetchMethod):
53 53
54 ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d) 54 ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d)
55 55
56 def forcefetch(self, url, ud, d): 56 def need_update(self, url, ud, d):
57 return ud.date == "now" 57 if ud.date == "now":
58 return True
59 if not os.path.exists(ud.localpath):
60 return True
61 return False
58 62
59 def download(self, loc, ud, d): 63 def download(self, loc, ud, d):
60 """Fetch urls""" 64 """Fetch urls"""