summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--bitbake/lib/bb/fetch/__init__.py94
-rw-r--r--bitbake/lib/bb/fetch/bzr.py5
-rw-r--r--bitbake/lib/bb/fetch/cvs.py5
-rw-r--r--bitbake/lib/bb/fetch/git.py9
-rw-r--r--bitbake/lib/bb/fetch/hg.py5
-rw-r--r--bitbake/lib/bb/fetch/osc.py5
-rw-r--r--bitbake/lib/bb/fetch/perforce.py5
-rw-r--r--bitbake/lib/bb/fetch/svk.py3
-rw-r--r--bitbake/lib/bb/fetch/svn.py5
-rw-r--r--bitbake/lib/bb/fetch/wget.py16
10 files changed, 53 insertions, 99 deletions
diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py
index ab4658bc3b..ccb60de59c 100644
--- a/bitbake/lib/bb/fetch/__init__.py
+++ b/bitbake/lib/bb/fetch/__init__.py
@@ -185,7 +185,18 @@ def go(d, urls = None):
185 pass 185 pass
186 bb.utils.unlockfile(lf) 186 bb.utils.unlockfile(lf)
187 continue 187 continue
188 m.go(u, ud, d) 188
189 # First try fetching uri, u, from PREMIRRORS
190 mirrors = [ i.split() for i in (bb.data.getVar('PREMIRRORS', d, 1) or "").split('\n') if i ]
191 if not try_mirrors(d, u, mirrors):
192 # Next try fetching from the original uri, u
193 try:
194 m.go(u, ud, d)
195 except:
196 # Finally, try fetching uri, u, from MIRRORS
197 mirrors = [ i.split() for i in (bb.data.getVar('MIRRORS', d, 1) or "").split('\n') if i ]
198 try_mirrors (d, u, mirrors)
199
189 if ud.localfile: 200 if ud.localfile:
190 if not m.forcefetch(u, ud, d): 201 if not m.forcefetch(u, ud, d):
191 Fetch.write_md5sum(u, ud, d) 202 Fetch.write_md5sum(u, ud, d)
@@ -332,6 +343,44 @@ def runfetchcmd(cmd, d, quiet = False):
332 343
333 return output 344 return output
334 345
346def try_mirrors(d, uri, mirrors):
347 """
348 Try to use a mirrored version of the sources.
349 This method will be automatically called before the fetchers go.
350
351 d Is a bb.data instance
352 uri is the original uri we're trying to download
353 mirrors is the list of mirrors we're going to try
354 """
355 fpath = os.path.join(data.getVar("DL_DIR", d, 1), os.path.basename(uri))
356 if os.access(fpath, os.R_OK):
357 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % fpath)
358 return True
359
360 ld = d.createCopy()
361 for (find, replace) in mirrors:
362 newuri = uri_replace(uri, find, replace, ld)
363 if newuri != uri:
364 try:
365 ud = FetchData(newuri, ld)
366 except bb.fetch.NoMethodError:
367 bb.msg.debug(1, bb.msg.domain.Fetcher, "No method for %s" % url)
368 continue
369
370 ud.setup_localpath(ld)
371
372 try:
373 ud.method.go(newuri, ud, ld)
374 return True
375 except (bb.fetch.MissingParameterError,
376 bb.fetch.FetchError,
377 bb.fetch.MD5SumError):
378 import sys
379 (type, value, traceback) = sys.exc_info()
380 bb.msg.debug(2, bb.msg.domain.Fetcher, "Mirror fetch failure: %s" % value)
381 return False
382
383
335class FetchData(object): 384class FetchData(object):
336 """ 385 """
337 A class which represents the fetcher state for a given URI. 386 A class which represents the fetcher state for a given URI.
@@ -489,49 +538,6 @@ class Fetch(object):
489 538
490 localcount_internal_helper = staticmethod(localcount_internal_helper) 539 localcount_internal_helper = staticmethod(localcount_internal_helper)
491 540
492 def try_mirror(d, tarfn):
493 """
494 Try to use a mirrored version of the sources. We do this
495 to avoid massive loads on foreign cvs and svn servers.
496 This method will be used by the different fetcher
497 implementations.
498
499 d Is a bb.data instance
500 tarfn is the name of the tarball
501 """
502 tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn)
503 if os.access(tarpath, os.R_OK):
504 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % tarfn)
505 return True
506
507 pn = data.getVar('PN', d, True)
508 src_tarball_stash = None
509 if pn:
510 src_tarball_stash = (data.getVar('SRC_TARBALL_STASH_%s' % pn, d, True) or data.getVar('CVS_TARBALL_STASH_%s' % pn, d, True) or data.getVar('SRC_TARBALL_STASH', d, True) or data.getVar('CVS_TARBALL_STASH', d, True) or "").split()
511
512 ld = d.createCopy()
513 for stash in src_tarball_stash:
514 url = stash + tarfn
515 try:
516 ud = FetchData(url, ld)
517 except bb.fetch.NoMethodError:
518 bb.msg.debug(1, bb.msg.domain.Fetcher, "No method for %s" % url)
519 continue
520
521 ud.setup_localpath(ld)
522
523 try:
524 ud.method.go(url, ud, ld)
525 return True
526 except (bb.fetch.MissingParameterError,
527 bb.fetch.FetchError,
528 bb.fetch.MD5SumError):
529 import sys
530 (type, value, traceback) = sys.exc_info()
531 bb.msg.debug(2, bb.msg.domain.Fetcher, "Tarball stash fetch failure: %s" % value)
532 return False
533 try_mirror = staticmethod(try_mirror)
534
535 def verify_md5sum(ud, got_sum): 541 def verify_md5sum(ud, got_sum):
536 """ 542 """
537 Verify the md5sum we wanted with the one we got 543 Verify the md5sum we wanted with the one we got
diff --git a/bitbake/lib/bb/fetch/bzr.py b/bitbake/lib/bb/fetch/bzr.py
index b27fb63d07..c6e33c3343 100644
--- a/bitbake/lib/bb/fetch/bzr.py
+++ b/bitbake/lib/bb/fetch/bzr.py
@@ -91,11 +91,6 @@ class Bzr(Fetch):
91 def go(self, loc, ud, d): 91 def go(self, loc, ud, d):
92 """Fetch url""" 92 """Fetch url"""
93 93
94 # try to use the tarball stash
95 if Fetch.try_mirror(d, ud.localfile):
96 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping bzr checkout." % ud.localpath)
97 return
98
99 if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK): 94 if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
100 bzrcmd = self._buildbzrcommand(ud, d, "update") 95 bzrcmd = self._buildbzrcommand(ud, d, "update")
101 bb.msg.debug(1, bb.msg.domain.Fetcher, "BZR Update %s" % loc) 96 bb.msg.debug(1, bb.msg.domain.Fetcher, "BZR Update %s" % loc)
diff --git a/bitbake/lib/bb/fetch/cvs.py b/bitbake/lib/bb/fetch/cvs.py
index 90a006500e..443f521317 100644
--- a/bitbake/lib/bb/fetch/cvs.py
+++ b/bitbake/lib/bb/fetch/cvs.py
@@ -77,11 +77,6 @@ class Cvs(Fetch):
77 77
78 def go(self, loc, ud, d): 78 def go(self, loc, ud, d):
79 79
80 # try to use the tarball stash
81 if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
82 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping cvs checkout." % ud.localpath)
83 return
84
85 method = "pserver" 80 method = "pserver"
86 if "method" in ud.parm: 81 if "method" in ud.parm:
87 method = ud.parm["method"] 82 method = ud.parm["method"]
diff --git a/bitbake/lib/bb/fetch/git.py b/bitbake/lib/bb/fetch/git.py
index 9430582d55..10b396bb00 100644
--- a/bitbake/lib/bb/fetch/git.py
+++ b/bitbake/lib/bb/fetch/git.py
@@ -82,10 +82,6 @@ class Git(Fetch):
82 def go(self, loc, ud, d): 82 def go(self, loc, ud, d):
83 """Fetch url""" 83 """Fetch url"""
84 84
85 if Fetch.try_mirror(d, ud.localfile):
86 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath)
87 return
88
89 if ud.user: 85 if ud.user:
90 username = ud.user + '@' 86 username = ud.user + '@'
91 else: 87 else:
@@ -97,11 +93,12 @@ class Git(Fetch):
97 codir = os.path.join(ud.clonedir, coname) 93 codir = os.path.join(ud.clonedir, coname)
98 94
99 if not os.path.exists(ud.clonedir): 95 if not os.path.exists(ud.clonedir):
100 if Fetch.try_mirror(d, ud.mirrortarball): 96 try:
97 Fetch.try_mirrors(ud.mirrortarball)
101 bb.mkdirhier(ud.clonedir) 98 bb.mkdirhier(ud.clonedir)
102 os.chdir(ud.clonedir) 99 os.chdir(ud.clonedir)
103 runfetchcmd("tar -xzf %s" % (repofile), d) 100 runfetchcmd("tar -xzf %s" % (repofile), d)
104 else: 101 except:
105 runfetchcmd("git clone -n %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.clonedir), d) 102 runfetchcmd("git clone -n %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.clonedir), d)
106 103
107 os.chdir(ud.clonedir) 104 os.chdir(ud.clonedir)
diff --git a/bitbake/lib/bb/fetch/hg.py b/bitbake/lib/bb/fetch/hg.py
index 08cb61fc28..d0756382f8 100644
--- a/bitbake/lib/bb/fetch/hg.py
+++ b/bitbake/lib/bb/fetch/hg.py
@@ -116,11 +116,6 @@ class Hg(Fetch):
116 def go(self, loc, ud, d): 116 def go(self, loc, ud, d):
117 """Fetch url""" 117 """Fetch url"""
118 118
119 # try to use the tarball stash
120 if Fetch.try_mirror(d, ud.localfile):
121 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping hg checkout." % ud.localpath)
122 return
123
124 bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'") 119 bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
125 120
126 if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK): 121 if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
diff --git a/bitbake/lib/bb/fetch/osc.py b/bitbake/lib/bb/fetch/osc.py
index 2c34caf6c9..548dd9d074 100644
--- a/bitbake/lib/bb/fetch/osc.py
+++ b/bitbake/lib/bb/fetch/osc.py
@@ -91,11 +91,6 @@ class Osc(Fetch):
91 Fetch url 91 Fetch url
92 """ 92 """
93 93
94 # Try to use the tarball stash
95 if Fetch.try_mirror(d, ud.localfile):
96 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping osc checkout." % ud.localpath)
97 return
98
99 bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'") 94 bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
100 95
101 if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK): 96 if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK):
diff --git a/bitbake/lib/bb/fetch/perforce.py b/bitbake/lib/bb/fetch/perforce.py
index 394f5a2253..e2c3421089 100644
--- a/bitbake/lib/bb/fetch/perforce.py
+++ b/bitbake/lib/bb/fetch/perforce.py
@@ -124,11 +124,6 @@ class Perforce(Fetch):
124 Fetch urls 124 Fetch urls
125 """ 125 """
126 126
127 # try to use the tarball stash
128 if Fetch.try_mirror(d, ud.localfile):
129 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping perforce checkout." % ud.localpath)
130 return
131
132 (host,depot,user,pswd,parm) = Perforce.doparse(loc, d) 127 (host,depot,user,pswd,parm) = Perforce.doparse(loc, d)
133 128
134 if depot.find('/...') != -1: 129 if depot.find('/...') != -1:
diff --git a/bitbake/lib/bb/fetch/svk.py b/bitbake/lib/bb/fetch/svk.py
index 120dad9d4e..a17ac04d21 100644
--- a/bitbake/lib/bb/fetch/svk.py
+++ b/bitbake/lib/bb/fetch/svk.py
@@ -62,9 +62,6 @@ class Svk(Fetch):
62 def go(self, loc, ud, d): 62 def go(self, loc, ud, d):
63 """Fetch urls""" 63 """Fetch urls"""
64 64
65 if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
66 return
67
68 svkroot = ud.host + ud.path 65 svkroot = ud.host + ud.path
69 66
70 svkcmd = "svk co -r {%s} %s/%s" % (ud.date, svkroot, ud.module) 67 svkcmd = "svk co -r {%s} %s/%s" % (ud.date, svkroot, ud.module)
diff --git a/bitbake/lib/bb/fetch/svn.py b/bitbake/lib/bb/fetch/svn.py
index eef9862a84..ba9f6ab109 100644
--- a/bitbake/lib/bb/fetch/svn.py
+++ b/bitbake/lib/bb/fetch/svn.py
@@ -136,11 +136,6 @@ class Svn(Fetch):
136 def go(self, loc, ud, d): 136 def go(self, loc, ud, d):
137 """Fetch url""" 137 """Fetch url"""
138 138
139 # try to use the tarball stash
140 if Fetch.try_mirror(d, ud.localfile):
141 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping svn checkout." % ud.localpath)
142 return
143
144 bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'") 139 bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
145 140
146 if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK): 141 if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
diff --git a/bitbake/lib/bb/fetch/wget.py b/bitbake/lib/bb/fetch/wget.py
index fd93c7ec46..ae1c6ad136 100644
--- a/bitbake/lib/bb/fetch/wget.py
+++ b/bitbake/lib/bb/fetch/wget.py
@@ -30,7 +30,6 @@ import bb
30from bb import data 30from bb import data
31from bb.fetch import Fetch 31from bb.fetch import Fetch
32from bb.fetch import FetchError 32from bb.fetch import FetchError
33from bb.fetch import uri_replace
34 33
35class Wget(Fetch): 34class Wget(Fetch):
36 """Class to fetch urls via 'wget'""" 35 """Class to fetch urls via 'wget'"""
@@ -105,24 +104,9 @@ class Wget(Fetch):
105 data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata) 104 data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata)
106 data.update_data(localdata) 105 data.update_data(localdata)
107 106
108 premirrors = [ i.split() for i in (data.getVar('PREMIRRORS', localdata, 1) or "").split('\n') if i ]
109 for (find, replace) in premirrors:
110 newuri = uri_replace(uri, find, replace, d)
111 if newuri != uri:
112 if fetch_uri(newuri, ud, localdata):
113 return True
114
115 if fetch_uri(uri, ud, localdata): 107 if fetch_uri(uri, ud, localdata):
116 return True 108 return True
117 109
118 # try mirrors
119 mirrors = [ i.split() for i in (data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ]
120 for (find, replace) in mirrors:
121 newuri = uri_replace(uri, find, replace, d)
122 if newuri != uri:
123 if fetch_uri(newuri, ud, localdata):
124 return True
125
126 raise FetchError(uri) 110 raise FetchError(uri)
127 111
128 112