diff options
author | Richard Purdie <richard@openedhand.com> | 2007-08-03 13:40:52 +0000 |
---|---|---|
committer | Richard Purdie <richard@openedhand.com> | 2007-08-03 13:40:52 +0000 |
commit | bfc70eb24e3ded25007811b1531673fa70b02401 (patch) | |
tree | fa4a29290d3178937fa085c147e8a51f815c6fdc /bitbake/lib/bb/fetch | |
parent | 034bbb805be0002fe6d689abde19662868b57b2c (diff) | |
download | poky-bfc70eb24e3ded25007811b1531673fa70b02401.tar.gz |
bitbake: Update along 1.8 branch
git-svn-id: https://svn.o-hand.com/repos/poky/trunk@2345 311d38ba-8fff-0310-9ca6-ca027cbcb966
Diffstat (limited to 'bitbake/lib/bb/fetch')
-rw-r--r-- | bitbake/lib/bb/fetch/__init__.py | 301 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch/git.py | 70 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch/perforce.py | 2 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch/svn.py | 177 |
4 files changed, 392 insertions, 158 deletions
diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py index 31a4adccb1..6ebf5a34ad 100644 --- a/bitbake/lib/bb/fetch/__init__.py +++ b/bitbake/lib/bb/fetch/__init__.py | |||
@@ -27,6 +27,12 @@ BitBake build tools. | |||
27 | import os, re | 27 | import os, re |
28 | import bb | 28 | import bb |
29 | from bb import data | 29 | from bb import data |
30 | from bb import persist_data | ||
31 | |||
32 | try: | ||
33 | import cPickle as pickle | ||
34 | except ImportError: | ||
35 | import pickle | ||
30 | 36 | ||
31 | class FetchError(Exception): | 37 | class FetchError(Exception): |
32 | """Exception raised when a download fails""" | 38 | """Exception raised when a download fails""" |
@@ -74,78 +80,193 @@ def uri_replace(uri, uri_find, uri_replace, d): | |||
74 | return bb.encodeurl(result_decoded) | 80 | return bb.encodeurl(result_decoded) |
75 | 81 | ||
76 | methods = [] | 82 | methods = [] |
77 | urldata = {} | ||
78 | |||
79 | def init(urls = [], d = None): | ||
80 | if d == None: | ||
81 | bb.msg.debug(2, bb.msg.domain.Fetcher, "BUG init called with None as data object!!!") | ||
82 | return | ||
83 | |||
84 | for m in methods: | ||
85 | m.urls = [] | ||
86 | 83 | ||
87 | for u in urls: | 84 | def fetcher_init(d): |
88 | ud = initdata(u, d) | 85 | """ |
89 | if ud.method: | 86 | Called to initilize the fetchers once the configuration data is known |
90 | ud.method.urls.append(u) | 87 | Calls before this must not hit the cache. |
91 | 88 | """ | |
92 | def initdata(url, d): | 89 | pd = persist_data.PersistData(d) |
93 | fn = bb.data.getVar('FILE', d, 1) | 90 | # Clear any cached url data |
94 | if fn not in urldata: | 91 | pd.delDomain("BB_URLDATA") |
95 | urldata[fn] = {} | 92 | # When to drop SCM head revisions should be controled by user policy |
96 | if url not in urldata[fn]: | 93 | pd.delDomain("BB_URI_HEADREVS") |
97 | ud = FetchData() | 94 | # Make sure our domains exist |
98 | (ud.type, ud.host, ud.path, ud.user, ud.pswd, ud.parm) = bb.decodeurl(data.expand(url, d)) | 95 | pd.addDomain("BB_URLDATA") |
99 | ud.date = Fetch.getSRCDate(ud, d) | 96 | pd.addDomain("BB_URI_HEADREVS") |
100 | for m in methods: | 97 | pd.addDomain("BB_URI_LOCALCOUNT") |
101 | if m.supports(url, ud, d): | 98 | |
102 | ud.localpath = m.localpath(url, ud, d) | 99 | # Function call order is usually: |
103 | ud.md5 = ud.localpath + '.md5' | 100 | # 1. init |
104 | # if user sets localpath for file, use it instead. | 101 | # 2. go |
105 | if "localpath" in ud.parm: | 102 | # 3. localpaths |
106 | ud.localpath = ud.parm["localpath"] | 103 | # localpath can be called at any time |
107 | ud.method = m | 104 | |
108 | break | 105 | def init(urls, d, cache = True): |
109 | urldata[fn][url] = ud | 106 | urldata = {} |
110 | return urldata[fn][url] | 107 | |
111 | 108 | if cache: | |
112 | def go(d): | 109 | urldata, pd, fn = getdata(d) |
113 | """Fetch all urls""" | 110 | |
111 | for url in urls: | ||
112 | if url not in urldata: | ||
113 | ud = FetchData(url, d) | ||
114 | for m in methods: | ||
115 | if m.supports(url, ud, d): | ||
116 | ud.init(m, d) | ||
117 | ud.setup_localpath(d) | ||
118 | break | ||
119 | urldata[url] = ud | ||
120 | |||
121 | if cache: | ||
122 | pd.setValue("BB_URLDATA", fn, pickle.dumps(urldata, 0)) | ||
123 | |||
124 | return urldata | ||
125 | |||
126 | def getdata(d): | ||
127 | urldata = {} | ||
114 | fn = bb.data.getVar('FILE', d, 1) | 128 | fn = bb.data.getVar('FILE', d, 1) |
115 | for m in methods: | 129 | pd = persist_data.PersistData(d) |
116 | for u in m.urls: | 130 | encdata = pd.getValue("BB_URLDATA", fn) |
117 | ud = urldata[fn][u] | 131 | if encdata: |
118 | if ud.localfile and not m.forcefetch(u, ud, d) and os.path.exists(urldata[fn][u].md5): | 132 | urldata = pickle.loads(str(encdata)) |
119 | # File already present along with md5 stamp file | 133 | |
120 | # Touch md5 file to show activity | 134 | return urldata, pd, fn |
121 | os.utime(ud.md5, None) | 135 | |
122 | continue | 136 | def go(d, urldata = None): |
123 | # RP - is olddir needed? | 137 | """ |
124 | # olddir = os.path.abspath(os.getcwd()) | 138 | Fetch all urls |
125 | m.go(u, ud , d) | 139 | """ |
126 | # os.chdir(olddir) | 140 | if not urldata: |
127 | if ud.localfile and not m.forcefetch(u, ud, d): | 141 | urldata, pd, fn = getdata(d) |
128 | Fetch.write_md5sum(u, ud, d) | 142 | |
129 | 143 | for u in urldata: | |
130 | def localpaths(d): | 144 | ud = urldata[u] |
131 | """Return a list of the local filenames, assuming successful fetch""" | 145 | m = ud.method |
146 | if ud.localfile and not m.forcefetch(u, ud, d) and os.path.exists(ud.md5): | ||
147 | # File already present along with md5 stamp file | ||
148 | # Touch md5 file to show activity | ||
149 | os.utime(ud.md5, None) | ||
150 | continue | ||
151 | m.go(u, ud, d) | ||
152 | if ud.localfile and not m.forcefetch(u, ud, d): | ||
153 | Fetch.write_md5sum(u, ud, d) | ||
154 | |||
155 | def localpaths(d, urldata = None): | ||
156 | """ | ||
157 | Return a list of the local filenames, assuming successful fetch | ||
158 | """ | ||
132 | local = [] | 159 | local = [] |
133 | fn = bb.data.getVar('FILE', d, 1) | 160 | if not urldata: |
134 | for m in methods: | 161 | urldata, pd, fn = getdata(d) |
135 | for u in m.urls: | 162 | |
136 | local.append(urldata[fn][u].localpath) | 163 | for u in urldata: |
164 | ud = urldata[u] | ||
165 | local.append(ud.localpath) | ||
166 | |||
137 | return local | 167 | return local |
138 | 168 | ||
139 | def localpath(url, d): | 169 | def get_srcrev(d): |
140 | ud = initdata(url, d) | 170 | """ |
141 | if ud.method: | 171 | Return the version string for the current package |
142 | return ud.localpath | 172 | (usually to be used as PV) |
173 | Most packages usually only have one SCM so we just pass on the call. | ||
174 | In the multi SCM case, we build a value based on SRCREV_FORMAT which must | ||
175 | have been set. | ||
176 | """ | ||
177 | scms = [] | ||
178 | urldata, pd, fn = getdata(d) | ||
179 | if len(urldata) == 0: | ||
180 | src_uri = bb.data.getVar('SRC_URI', d, 1).split() | ||
181 | for url in src_uri: | ||
182 | if url not in urldata: | ||
183 | ud = FetchData(url, d) | ||
184 | for m in methods: | ||
185 | if m.supports(url, ud, d): | ||
186 | ud.init(m, d) | ||
187 | break | ||
188 | urldata[url] = ud | ||
189 | if ud.method.suppports_srcrev(): | ||
190 | scms.append(url) | ||
191 | ud.setup_localpath(d) | ||
192 | else: | ||
193 | for u in urldata: | ||
194 | ud = urldata[u] | ||
195 | if ud.method.suppports_srcrev(): | ||
196 | scms.append(u) | ||
197 | |||
198 | if len(scms) == 0: | ||
199 | bb.msg.error(bb.msg.domain.Fetcher, "SRCREV was used yet no valid SCM was found in SRC_URI") | ||
200 | raise ParameterError | ||
201 | |||
202 | if len(scms) == 1: | ||
203 | return urldata[scms[0]].method.sortable_revision(scms[0], urldata[scms[0]], d) | ||
204 | |||
205 | bb.msg.error(bb.msg.domain.Fetcher, "Sorry, support for SRCREV_FORMAT still needs to be written") | ||
206 | raise ParameterError | ||
207 | |||
208 | def localpath(url, d, cache = True): | ||
209 | """ | ||
210 | Called from the parser with cache=False since the cache isn't ready | ||
211 | at this point. Also called from classed in OE e.g. patch.bbclass | ||
212 | """ | ||
213 | ud = init([url], d, cache) | ||
214 | if ud[url].method: | ||
215 | return ud[url].localpath | ||
143 | return url | 216 | return url |
144 | 217 | ||
218 | def runfetchcmd(cmd, d, quiet = False): | ||
219 | """ | ||
220 | Run cmd returning the command output | ||
221 | Raise an error if interrupted or cmd fails | ||
222 | Optionally echo command output to stdout | ||
223 | """ | ||
224 | bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % cmd) | ||
225 | |||
226 | # Need to export PATH as binary could be in metadata paths | ||
227 | # rather than host provided | ||
228 | pathcmd = 'export PATH=%s; %s' % (data.expand('${PATH}', d), cmd) | ||
229 | |||
230 | stdout_handle = os.popen(pathcmd, "r") | ||
231 | output = "" | ||
232 | |||
233 | while 1: | ||
234 | line = stdout_handle.readline() | ||
235 | if not line: | ||
236 | break | ||
237 | if not quiet: | ||
238 | print line | ||
239 | output += line | ||
240 | |||
241 | status = stdout_handle.close() or 0 | ||
242 | signal = status >> 8 | ||
243 | exitstatus = status & 0xff | ||
244 | |||
245 | if signal: | ||
246 | raise FetchError("Fetch command %s failed with signal %s, output:\n%s" % (pathcmd, signal, output)) | ||
247 | elif status != 0: | ||
248 | raise FetchError("Fetch command %s failed with exit code %s, output:\n%s" % (pathcmd, status, output)) | ||
249 | |||
250 | return output | ||
251 | |||
145 | class FetchData(object): | 252 | class FetchData(object): |
146 | """Class for fetcher variable store""" | 253 | """Class for fetcher variable store""" |
147 | def __init__(self): | 254 | def __init__(self, url, d): |
148 | self.localfile = "" | 255 | self.localfile = "" |
256 | (self.type, self.host, self.path, self.user, self.pswd, self.parm) = bb.decodeurl(data.expand(url, d)) | ||
257 | self.date = Fetch.getSRCDate(self, d) | ||
258 | self.url = url | ||
259 | |||
260 | def init(self, method, d): | ||
261 | self.method = method | ||
262 | |||
263 | def setup_localpath(self, d): | ||
264 | if "localpath" in self.parm: | ||
265 | self.localpath = self.parm["localpath"] | ||
266 | else: | ||
267 | self.localpath = self.method.localpath(self.url, self, d) | ||
268 | self.md5 = self.localpath + '.md5' | ||
269 | # if user sets localpath for file, use it instead. | ||
149 | 270 | ||
150 | 271 | ||
151 | class Fetch(object): | 272 | class Fetch(object): |
@@ -182,6 +303,12 @@ class Fetch(object): | |||
182 | """ | 303 | """ |
183 | return False | 304 | return False |
184 | 305 | ||
306 | def suppports_srcrev(self): | ||
307 | """ | ||
308 | The fetcher supports auto source revisions (SRCREV) | ||
309 | """ | ||
310 | return False | ||
311 | |||
185 | def go(self, url, urldata, d): | 312 | def go(self, url, urldata, d): |
186 | """ | 313 | """ |
187 | Fetch urls | 314 | Fetch urls |
@@ -269,6 +396,50 @@ class Fetch(object): | |||
269 | md5out.close() | 396 | md5out.close() |
270 | write_md5sum = staticmethod(write_md5sum) | 397 | write_md5sum = staticmethod(write_md5sum) |
271 | 398 | ||
399 | def latest_revision(self, url, ud, d): | ||
400 | """ | ||
401 | Look in the cache for the latest revision, if not present ask the SCM. | ||
402 | """ | ||
403 | if not hasattr(self, "_latest_revision"): | ||
404 | raise ParameterError | ||
405 | |||
406 | pd = persist_data.PersistData(d) | ||
407 | key = self._revision_key(url, ud, d) | ||
408 | rev = pd.getValue("BB_URI_HEADREVS", key) | ||
409 | if rev != None: | ||
410 | return str(rev) | ||
411 | |||
412 | rev = self._latest_revision(url, ud, d) | ||
413 | pd.setValue("BB_URI_HEADREVS", key, rev) | ||
414 | return rev | ||
415 | |||
416 | def sortable_revision(self, url, ud, d): | ||
417 | """ | ||
418 | |||
419 | """ | ||
420 | if hasattr(self, "_sortable_revision"): | ||
421 | return self._sortable_revision(url, ud, d) | ||
422 | |||
423 | pd = persist_data.PersistData(d) | ||
424 | key = self._revision_key(url, ud, d) | ||
425 | latest_rev = self.latest_revision(url, ud, d) | ||
426 | last_rev = pd.getValue("BB_URI_LOCALCOUNT", key + "_rev") | ||
427 | count = pd.getValue("BB_URI_LOCALCOUNT", key + "_count") | ||
428 | |||
429 | if last_rev == latest_rev: | ||
430 | return str(count + "+" + latest_rev) | ||
431 | |||
432 | if count is None: | ||
433 | count = "0" | ||
434 | else: | ||
435 | count = str(int(count) + 1) | ||
436 | |||
437 | pd.setValue("BB_URI_LOCALCOUNT", key + "_rev", latest_rev) | ||
438 | pd.setValue("BB_URI_LOCALCOUNT", key + "_count", count) | ||
439 | |||
440 | return str(count + "+" + latest_rev) | ||
441 | |||
442 | |||
272 | import cvs | 443 | import cvs |
273 | import git | 444 | import git |
274 | import local | 445 | import local |
@@ -278,11 +449,11 @@ import svk | |||
278 | import ssh | 449 | import ssh |
279 | import perforce | 450 | import perforce |
280 | 451 | ||
281 | methods.append(cvs.Cvs()) | ||
282 | methods.append(git.Git()) | ||
283 | methods.append(local.Local()) | 452 | methods.append(local.Local()) |
284 | methods.append(svn.Svn()) | ||
285 | methods.append(wget.Wget()) | 453 | methods.append(wget.Wget()) |
454 | methods.append(svn.Svn()) | ||
455 | methods.append(git.Git()) | ||
456 | methods.append(cvs.Cvs()) | ||
286 | methods.append(svk.Svk()) | 457 | methods.append(svk.Svk()) |
287 | methods.append(ssh.SSH()) | 458 | methods.append(ssh.SSH()) |
288 | methods.append(perforce.Perforce()) | 459 | methods.append(perforce.Perforce()) |
diff --git a/bitbake/lib/bb/fetch/git.py b/bitbake/lib/bb/fetch/git.py index c0cd27df09..7d55ee9138 100644 --- a/bitbake/lib/bb/fetch/git.py +++ b/bitbake/lib/bb/fetch/git.py | |||
@@ -25,6 +25,7 @@ import bb | |||
25 | from bb import data | 25 | from bb import data |
26 | from bb.fetch import Fetch | 26 | from bb.fetch import Fetch |
27 | from bb.fetch import FetchError | 27 | from bb.fetch import FetchError |
28 | from bb.fetch import runfetchcmd | ||
28 | 29 | ||
29 | def prunedir(topdir): | 30 | def prunedir(topdir): |
30 | # Delete everything reachable from the directory named in 'topdir'. | 31 | # Delete everything reachable from the directory named in 'topdir'. |
@@ -35,19 +36,6 @@ def prunedir(topdir): | |||
35 | for name in dirs: | 36 | for name in dirs: |
36 | os.rmdir(os.path.join(root, name)) | 37 | os.rmdir(os.path.join(root, name)) |
37 | 38 | ||
38 | def rungitcmd(cmd,d): | ||
39 | |||
40 | bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % cmd) | ||
41 | |||
42 | # Need to export PATH as git is likely to be in metadata paths | ||
43 | # rather than host provided | ||
44 | pathcmd = 'export PATH=%s; %s' % (data.expand('${PATH}', d), cmd) | ||
45 | |||
46 | myret = os.system(pathcmd) | ||
47 | |||
48 | if myret != 0: | ||
49 | raise FetchError("Git: %s failed" % pathcmd) | ||
50 | |||
51 | class Git(Fetch): | 39 | class Git(Fetch): |
52 | """Class to fetch a module or modules from git repositories""" | 40 | """Class to fetch a module or modules from git repositories""" |
53 | def supports(self, url, ud, d): | 41 | def supports(self, url, ud, d): |
@@ -62,24 +50,22 @@ class Git(Fetch): | |||
62 | if 'protocol' in ud.parm: | 50 | if 'protocol' in ud.parm: |
63 | ud.proto = ud.parm['protocol'] | 51 | ud.proto = ud.parm['protocol'] |
64 | 52 | ||
65 | ud.tag = "master" | 53 | tag = data.getVar("SRCREV", d, 0) |
66 | if 'tag' in ud.parm: | 54 | if 'tag' in ud.parm: |
67 | ud.tag = ud.parm['tag'] | 55 | ud.tag = ud.parm['tag'] |
56 | elif tag and "get_srcrev" not in tag and len(tag) == 40: | ||
57 | ud.tag = tag | ||
58 | else: | ||
59 | ud.tag = self.latest_revision(url, ud, d) | ||
68 | 60 | ||
69 | ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.tag), d) | 61 | ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.tag), d) |
70 | 62 | ||
71 | return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) | 63 | return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) |
72 | 64 | ||
73 | def forcefetch(self, url, ud, d): | ||
74 | # tag=="master" must always update | ||
75 | if (ud.tag == "master"): | ||
76 | return True | ||
77 | return False | ||
78 | |||
79 | def go(self, loc, ud, d): | 65 | def go(self, loc, ud, d): |
80 | """Fetch url""" | 66 | """Fetch url""" |
81 | 67 | ||
82 | if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile): | 68 | if Fetch.try_mirror(d, ud.localfile): |
83 | bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath) | 69 | bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath) |
84 | return | 70 | return |
85 | 71 | ||
@@ -96,32 +82,50 @@ class Git(Fetch): | |||
96 | if Fetch.try_mirror(d, repofilename): | 82 | if Fetch.try_mirror(d, repofilename): |
97 | bb.mkdirhier(repodir) | 83 | bb.mkdirhier(repodir) |
98 | os.chdir(repodir) | 84 | os.chdir(repodir) |
99 | rungitcmd("tar -xzf %s" % (repofile),d) | 85 | runfetchcmd("tar -xzf %s" % (repofile), d) |
100 | else: | 86 | else: |
101 | rungitcmd("git clone -n %s://%s%s %s" % (ud.proto, ud.host, ud.path, repodir),d) | 87 | runfetchcmd("git clone -n %s://%s%s %s" % (ud.proto, ud.host, ud.path, repodir), d) |
102 | 88 | ||
103 | os.chdir(repodir) | 89 | os.chdir(repodir) |
104 | rungitcmd("git pull %s://%s%s" % (ud.proto, ud.host, ud.path),d) | ||
105 | rungitcmd("git pull --tags %s://%s%s" % (ud.proto, ud.host, ud.path),d) | ||
106 | rungitcmd("git prune-packed", d) | ||
107 | rungitcmd("git pack-redundant --all | xargs -r rm", d) | ||
108 | # Remove all but the .git directory | 90 | # Remove all but the .git directory |
109 | rungitcmd("rm * -Rf", d) | 91 | runfetchcmd("rm * -Rf", d) |
92 | runfetchcmd("git pull %s://%s%s" % (ud.proto, ud.host, ud.path), d) | ||
93 | runfetchcmd("git pull --tags %s://%s%s" % (ud.proto, ud.host, ud.path), d) | ||
94 | runfetchcmd("git prune-packed", d) | ||
95 | runfetchcmd("git pack-redundant --all | xargs -r rm", d) | ||
110 | # old method of downloading tags | 96 | # old method of downloading tags |
111 | #rungitcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (ud.host, ud.path, os.path.join(repodir, ".git", "")),d) | 97 | #runfetchcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (ud.host, ud.path, os.path.join(repodir, ".git", "")), d) |
112 | 98 | ||
113 | os.chdir(repodir) | 99 | os.chdir(repodir) |
114 | bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository") | 100 | bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository") |
115 | rungitcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ),d) | 101 | runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d) |
116 | 102 | ||
117 | if os.path.exists(codir): | 103 | if os.path.exists(codir): |
118 | prunedir(codir) | 104 | prunedir(codir) |
119 | 105 | ||
120 | bb.mkdirhier(codir) | 106 | bb.mkdirhier(codir) |
121 | os.chdir(repodir) | 107 | os.chdir(repodir) |
122 | rungitcmd("git read-tree %s" % (ud.tag),d) | 108 | runfetchcmd("git read-tree %s" % (ud.tag), d) |
123 | rungitcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")),d) | 109 | runfetchcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")), d) |
124 | 110 | ||
125 | os.chdir(codir) | 111 | os.chdir(codir) |
126 | bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout") | 112 | bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout") |
127 | rungitcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ),d) | 113 | runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d) |
114 | |||
115 | os.chdir(repodir) | ||
116 | prunedir(codir) | ||
117 | |||
118 | def suppports_srcrev(self): | ||
119 | return True | ||
120 | |||
121 | def _revision_key(self, url, ud, d): | ||
122 | """ | ||
123 | Return a unique key for the url | ||
124 | """ | ||
125 | return "git:" + ud.host + ud.path.replace('/', '.') | ||
126 | |||
127 | def _latest_revision(self, url, ud, d): | ||
128 | |||
129 | output = runfetchcmd("git ls-remote %s://%s%s" % (ud.proto, ud.host, ud.path), d, True) | ||
130 | return output.split()[0] | ||
131 | |||
diff --git a/bitbake/lib/bb/fetch/perforce.py b/bitbake/lib/bb/fetch/perforce.py index 125eb99aa6..97b618228b 100644 --- a/bitbake/lib/bb/fetch/perforce.py +++ b/bitbake/lib/bb/fetch/perforce.py | |||
@@ -125,7 +125,7 @@ class Perforce(Fetch): | |||
125 | """ | 125 | """ |
126 | 126 | ||
127 | # try to use the tarball stash | 127 | # try to use the tarball stash |
128 | if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile): | 128 | if Fetch.try_mirror(d, ud.localfile): |
129 | bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping perforce checkout." % ud.localpath) | 129 | bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping perforce checkout." % ud.localpath) |
130 | return | 130 | return |
131 | 131 | ||
diff --git a/bitbake/lib/bb/fetch/svn.py b/bitbake/lib/bb/fetch/svn.py index 120f4f8539..ca12efe158 100644 --- a/bitbake/lib/bb/fetch/svn.py +++ b/bitbake/lib/bb/fetch/svn.py | |||
@@ -1,17 +1,12 @@ | |||
1 | # ex:ts=4:sw=4:sts=4:et | 1 | # ex:ts=4:sw=4:sts=4:et |
2 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- | 2 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- |
3 | """ | 3 | """ |
4 | BitBake 'Fetch' implementations | 4 | BitBake 'Fetch' implementation for svn. |
5 | |||
6 | This implementation is for svn. It is based on the cvs implementation. | ||
7 | 5 | ||
8 | """ | 6 | """ |
9 | 7 | ||
10 | # Copyright (C) 2004 Marcin Juszkiewicz | 8 | # Copyright (C) 2003, 2004 Chris Larson |
11 | # | 9 | # Copyright (C) 2004 Marcin Juszkiewicz |
12 | # Classes for obtaining upstream sources for the | ||
13 | # BitBake build tools. | ||
14 | # Copyright (C) 2003, 2004 Chris Larson | ||
15 | # | 10 | # |
16 | # This program is free software; you can redistribute it and/or modify | 11 | # This program is free software; you can redistribute it and/or modify |
17 | # it under the terms of the GNU General Public License version 2 as | 12 | # it under the terms of the GNU General Public License version 2 as |
@@ -35,6 +30,7 @@ from bb import data | |||
35 | from bb.fetch import Fetch | 30 | from bb.fetch import Fetch |
36 | from bb.fetch import FetchError | 31 | from bb.fetch import FetchError |
37 | from bb.fetch import MissingParameterError | 32 | from bb.fetch import MissingParameterError |
33 | from bb.fetch import runfetchcmd | ||
38 | 34 | ||
39 | class Svn(Fetch): | 35 | class Svn(Fetch): |
40 | """Class to fetch a module or modules from svn repositories""" | 36 | """Class to fetch a module or modules from svn repositories""" |
@@ -47,32 +43,54 @@ class Svn(Fetch): | |||
47 | def localpath(self, url, ud, d): | 43 | def localpath(self, url, ud, d): |
48 | if not "module" in ud.parm: | 44 | if not "module" in ud.parm: |
49 | raise MissingParameterError("svn method needs a 'module' parameter") | 45 | raise MissingParameterError("svn method needs a 'module' parameter") |
50 | else: | ||
51 | ud.module = ud.parm["module"] | ||
52 | 46 | ||
53 | ud.revision = "" | 47 | ud.module = ud.parm["module"] |
54 | if 'rev' in ud.parm: | 48 | |
55 | ud.revision = ud.parm['rev'] | 49 | # Create paths to svn checkouts |
50 | relpath = ud.path | ||
51 | if relpath.startswith('/'): | ||
52 | # Remove leading slash as os.path.join can't cope | ||
53 | relpath = relpath[1:] | ||
54 | ud.pkgdir = os.path.join(data.expand('${SVNDIR}', d), ud.host, relpath) | ||
55 | ud.moddir = os.path.join(ud.pkgdir, ud.module) | ||
56 | 56 | ||
57 | if ud.revision: | 57 | if 'rev' in ud.parm: |
58 | ud.date = "" | 58 | ud.date = "" |
59 | ud.revision = ud.parm['rev'] | ||
60 | elif 'date' in ud.date: | ||
61 | ud.date = ud.parm['date'] | ||
62 | ud.revision = "" | ||
63 | else: | ||
64 | # | ||
65 | # ***Nasty hacks*** | ||
66 | # If DATE in unexpanded PV, use ud.date (which is set from SRCDATE) | ||
67 | # Will warn people to switch to SRCREV here | ||
68 | # | ||
69 | # How can we tell when a user has overriden SRCDATE? | ||
70 | # check for "get_srcdate" in unexpanded SRCREV - ugly | ||
71 | # | ||
72 | pv = data.getVar("PV", d, 0) | ||
73 | if "DATE" in pv: | ||
74 | ud.revision = "" | ||
75 | else: | ||
76 | rev = data.getVar("SRCREV", d, 0) | ||
77 | if "get_srcrev" in rev: | ||
78 | ud.revision = self.latest_revision(url, ud, d) | ||
79 | else: | ||
80 | ud.revision = rev | ||
81 | ud.date = "" | ||
59 | 82 | ||
60 | ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d) | 83 | ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d) |
61 | 84 | ||
62 | return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) | 85 | return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) |
63 | 86 | ||
64 | def forcefetch(self, url, ud, d): | 87 | def _buildsvncommand(self, ud, d, command): |
65 | if (ud.date == "now"): | 88 | """ |
66 | return True | 89 | Build up an svn commandline based on ud |
67 | return False | 90 | command is "fetch", "update", "info" |
68 | 91 | """ | |
69 | def go(self, loc, ud, d): | ||
70 | """Fetch url""" | ||
71 | 92 | ||
72 | # try to use the tarball stash | 93 | basecmd = data.expand('${FETCHCMD_svn}', d) |
73 | if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile): | ||
74 | bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping svn checkout." % ud.localpath) | ||
75 | return | ||
76 | 94 | ||
77 | proto = "svn" | 95 | proto = "svn" |
78 | if "proto" in ud.parm: | 96 | if "proto" in ud.parm: |
@@ -84,12 +102,8 @@ class Svn(Fetch): | |||
84 | 102 | ||
85 | svnroot = ud.host + ud.path | 103 | svnroot = ud.host + ud.path |
86 | 104 | ||
87 | # either use the revision, or SRCDATE in braces, or nothing for SRCDATE = "now" | 105 | # either use the revision, or SRCDATE in braces, |
88 | options = [] | 106 | options = [] |
89 | if ud.revision: | ||
90 | options.append("-r %s" % ud.revision) | ||
91 | elif ud.date != "now": | ||
92 | options.append("-r {%s}" % ud.date) | ||
93 | 107 | ||
94 | if ud.user: | 108 | if ud.user: |
95 | options.append("--username %s" % ud.user) | 109 | options.append("--username %s" % ud.user) |
@@ -97,48 +111,93 @@ class Svn(Fetch): | |||
97 | if ud.pswd: | 111 | if ud.pswd: |
98 | options.append("--password %s" % ud.pswd) | 112 | options.append("--password %s" % ud.pswd) |
99 | 113 | ||
100 | localdata = data.createCopy(d) | 114 | if command is "info": |
101 | data.setVar('OVERRIDES', "svn:%s" % data.getVar('OVERRIDES', localdata), localdata) | 115 | svncmd = "%s info %s %s://%s/%s/" % (basecmd, " ".join(options), proto, svnroot, ud.module) |
102 | data.update_data(localdata) | 116 | else: |
103 | 117 | if ud.revision: | |
104 | data.setVar('SVNROOT', "%s://%s/%s" % (proto, svnroot, ud.module), localdata) | 118 | options.append("-r %s" % ud.revision) |
105 | data.setVar('SVNCOOPTS', " ".join(options), localdata) | 119 | elif ud.date: |
106 | data.setVar('SVNMODULE', ud.module, localdata) | 120 | options.append("-r {%s}" % ud.date) |
107 | svncmd = data.getVar('FETCHCOMMAND', localdata, 1) | 121 | |
108 | svnupcmd = data.getVar('UPDATECOMMAND', localdata, 1) | 122 | if command is "fetch": |
123 | svncmd = "%s co %s %s://%s/%s %s" % (basecmd, " ".join(options), proto, svnroot, ud.module, ud.module) | ||
124 | elif command is "update": | ||
125 | svncmd = "%s update %s" % (basecmd, " ".join(options)) | ||
126 | else: | ||
127 | raise FetchError("Invalid svn command %s" % command) | ||
109 | 128 | ||
110 | if svn_rsh: | 129 | if svn_rsh: |
111 | svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd) | 130 | svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd) |
112 | svnupcmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svnupcmd) | ||
113 | 131 | ||
114 | pkg = data.expand('${PN}', d) | 132 | return svncmd |
115 | pkgdir = os.path.join(data.expand('${SVNDIR}', localdata), pkg) | 133 | |
116 | moddir = os.path.join(pkgdir, ud.module) | 134 | def go(self, loc, ud, d): |
117 | bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + moddir + "'") | 135 | """Fetch url""" |
118 | 136 | ||
119 | if os.access(os.path.join(moddir, '.svn'), os.R_OK): | 137 | # try to use the tarball stash |
138 | if Fetch.try_mirror(d, ud.localfile): | ||
139 | bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping svn checkout." % ud.localpath) | ||
140 | return | ||
141 | |||
142 | bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'") | ||
143 | |||
144 | if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK): | ||
145 | svnupdatecmd = self._buildsvncommand(ud, d, "update") | ||
120 | bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc) | 146 | bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc) |
121 | # update sources there | 147 | # update sources there |
122 | os.chdir(moddir) | 148 | os.chdir(ud.moddir) |
123 | bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnupcmd) | 149 | bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnupdatecmd) |
124 | myret = os.system(svnupcmd) | 150 | runfetchcmd(svnupdatecmd, d) |
125 | else: | 151 | else: |
152 | svnfetchcmd = self._buildsvncommand(ud, d, "fetch") | ||
126 | bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc) | 153 | bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc) |
127 | # check out sources there | 154 | # check out sources there |
128 | bb.mkdirhier(pkgdir) | 155 | bb.mkdirhier(ud.pkgdir) |
129 | os.chdir(pkgdir) | 156 | os.chdir(ud.pkgdir) |
130 | bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svncmd) | 157 | bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnfetchcmd) |
131 | myret = os.system(svncmd) | 158 | runfetchcmd(svnfetchcmd, d) |
132 | |||
133 | if myret != 0: | ||
134 | raise FetchError(ud.module) | ||
135 | 159 | ||
136 | os.chdir(pkgdir) | 160 | os.chdir(ud.pkgdir) |
137 | # tar them up to a defined filename | 161 | # tar them up to a defined filename |
138 | myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module))) | 162 | try: |
139 | if myret != 0: | 163 | runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module)), d) |
164 | except: | ||
165 | t, v, tb = sys.exc_info() | ||
140 | try: | 166 | try: |
141 | os.unlink(ud.localpath) | 167 | os.unlink(ud.localpath) |
142 | except OSError: | 168 | except OSError: |
143 | pass | 169 | pass |
144 | raise FetchError(ud.module) | 170 | raise t, v, tb |
171 | |||
172 | def suppports_srcrev(self): | ||
173 | return True | ||
174 | |||
175 | def _revision_key(self, url, ud, d): | ||
176 | """ | ||
177 | Return a unique key for the url | ||
178 | """ | ||
179 | return "svn:" + ud.moddir | ||
180 | |||
181 | def _latest_revision(self, url, ud, d): | ||
182 | """ | ||
183 | Return the latest upstream revision number | ||
184 | """ | ||
185 | bb.msg.debug(2, bb.msg.domain.Fetcher, "SVN fetcher hitting network for %s" % url) | ||
186 | |||
187 | output = runfetchcmd("LANG=C LC_ALL=C " + self._buildsvncommand(ud, d, "info"), d, True) | ||
188 | |||
189 | revision = None | ||
190 | for line in output.splitlines(): | ||
191 | if "Last Changed Rev" in line: | ||
192 | revision = line.split(":")[1].strip() | ||
193 | |||
194 | return revision | ||
195 | |||
196 | def _sortable_revision(self, url, ud, d): | ||
197 | """ | ||
198 | Return a sortable revision number which in our case is the revision number | ||
199 | (use the cached version to avoid network access) | ||
200 | """ | ||
201 | |||
202 | return self.latest_revision(url, ud, d) | ||
203 | |||