summaryrefslogtreecommitdiffstats
path: root/bitbake-dev/lib/bb/fetch
diff options
context:
space:
mode:
authorRichard Purdie <rpurdie@linux.intel.com>2010-01-20 18:46:02 +0000
committerRichard Purdie <rpurdie@linux.intel.com>2010-01-20 18:46:02 +0000
commit22c29d8651668195f72e2f6a8e059d625eb511c3 (patch)
treedd1dd43f0ec47a9964c8a766eb8b3ad75cf51a64 /bitbake-dev/lib/bb/fetch
parent1bfd6edef9db9c9175058ae801d1b601e4f15263 (diff)
downloadpoky-22c29d8651668195f72e2f6a8e059d625eb511c3.tar.gz
bitbake: Switch to bitbake-dev version (bitbake master upstream)
Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
Diffstat (limited to 'bitbake-dev/lib/bb/fetch')
-rw-r--r--bitbake-dev/lib/bb/fetch/__init__.py640
-rw-r--r--bitbake-dev/lib/bb/fetch/bzr.py153
-rw-r--r--bitbake-dev/lib/bb/fetch/cvs.py182
-rw-r--r--bitbake-dev/lib/bb/fetch/git.py216
-rw-r--r--bitbake-dev/lib/bb/fetch/hg.py178
-rw-r--r--bitbake-dev/lib/bb/fetch/local.py72
-rw-r--r--bitbake-dev/lib/bb/fetch/osc.py155
-rw-r--r--bitbake-dev/lib/bb/fetch/perforce.py214
-rw-r--r--bitbake-dev/lib/bb/fetch/ssh.py118
-rw-r--r--bitbake-dev/lib/bb/fetch/svk.py109
-rw-r--r--bitbake-dev/lib/bb/fetch/svn.py206
-rw-r--r--bitbake-dev/lib/bb/fetch/wget.py130
12 files changed, 0 insertions, 2373 deletions
diff --git a/bitbake-dev/lib/bb/fetch/__init__.py b/bitbake-dev/lib/bb/fetch/__init__.py
deleted file mode 100644
index ab4658bc3b..0000000000
--- a/bitbake-dev/lib/bb/fetch/__init__.py
+++ /dev/null
@@ -1,640 +0,0 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' implementations
5
6Classes for obtaining upstream sources for the
7BitBake build tools.
8"""
9
10# Copyright (C) 2003, 2004 Chris Larson
11#
12# This program is free software; you can redistribute it and/or modify
13# it under the terms of the GNU General Public License version 2 as
14# published by the Free Software Foundation.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License along
22# with this program; if not, write to the Free Software Foundation, Inc.,
23# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24#
25# Based on functions from the base bb module, Copyright 2003 Holger Schurig
26
27import os, re
28import bb
29from bb import data
30from bb import persist_data
31
32class FetchError(Exception):
33 """Exception raised when a download fails"""
34
35class NoMethodError(Exception):
36 """Exception raised when there is no method to obtain a supplied url or set of urls"""
37
38class MissingParameterError(Exception):
39 """Exception raised when a fetch method is missing a critical parameter in the url"""
40
41class ParameterError(Exception):
42 """Exception raised when a url cannot be proccessed due to invalid parameters."""
43
44class MD5SumError(Exception):
45 """Exception raised when a MD5SUM of a file does not match the expected one"""
46
47class InvalidSRCREV(Exception):
48 """Exception raised when an invalid SRCREV is encountered"""
49
50def uri_replace(uri, uri_find, uri_replace, d):
51# bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: operating on %s" % uri)
52 if not uri or not uri_find or not uri_replace:
53 bb.msg.debug(1, bb.msg.domain.Fetcher, "uri_replace: passed an undefined value, not replacing")
54 uri_decoded = list(bb.decodeurl(uri))
55 uri_find_decoded = list(bb.decodeurl(uri_find))
56 uri_replace_decoded = list(bb.decodeurl(uri_replace))
57 result_decoded = ['','','','','',{}]
58 for i in uri_find_decoded:
59 loc = uri_find_decoded.index(i)
60 result_decoded[loc] = uri_decoded[loc]
61 import types
62 if type(i) == types.StringType:
63 if (re.match(i, uri_decoded[loc])):
64 result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
65 if uri_find_decoded.index(i) == 2:
66 if d:
67 localfn = bb.fetch.localpath(uri, d)
68 if localfn:
69 result_decoded[loc] = os.path.dirname(result_decoded[loc]) + "/" + os.path.basename(bb.fetch.localpath(uri, d))
70# bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc]))
71 else:
72# bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: no match")
73 return uri
74# else:
75# for j in i.keys():
76# FIXME: apply replacements against options
77 return bb.encodeurl(result_decoded)
78
79methods = []
80urldata_cache = {}
81saved_headrevs = {}
82
83def fetcher_init(d):
84 """
85 Called to initilize the fetchers once the configuration data is known
86 Calls before this must not hit the cache.
87 """
88 pd = persist_data.PersistData(d)
89 # When to drop SCM head revisions controled by user policy
90 srcrev_policy = bb.data.getVar('BB_SRCREV_POLICY', d, 1) or "clear"
91 if srcrev_policy == "cache":
92 bb.msg.debug(1, bb.msg.domain.Fetcher, "Keeping SRCREV cache due to cache policy of: %s" % srcrev_policy)
93 elif srcrev_policy == "clear":
94 bb.msg.debug(1, bb.msg.domain.Fetcher, "Clearing SRCREV cache due to cache policy of: %s" % srcrev_policy)
95 try:
96 bb.fetch.saved_headrevs = pd.getKeyValues("BB_URI_HEADREVS")
97 except:
98 pass
99 pd.delDomain("BB_URI_HEADREVS")
100 else:
101 bb.msg.fatal(bb.msg.domain.Fetcher, "Invalid SRCREV cache policy of: %s" % srcrev_policy)
102
103 for m in methods:
104 if hasattr(m, "init"):
105 m.init(d)
106
107 # Make sure our domains exist
108 pd.addDomain("BB_URI_HEADREVS")
109 pd.addDomain("BB_URI_LOCALCOUNT")
110
111def fetcher_compare_revisons(d):
112 """
113 Compare the revisions in the persistant cache with current values and
114 return true/false on whether they've changed.
115 """
116
117 pd = persist_data.PersistData(d)
118 data = pd.getKeyValues("BB_URI_HEADREVS")
119 data2 = bb.fetch.saved_headrevs
120
121 changed = False
122 for key in data:
123 if key not in data2 or data2[key] != data[key]:
124 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s changed" % key)
125 changed = True
126 return True
127 else:
128 bb.msg.debug(2, bb.msg.domain.Fetcher, "%s did not change" % key)
129 return False
130
131# Function call order is usually:
132# 1. init
133# 2. go
134# 3. localpaths
135# localpath can be called at any time
136
137def init(urls, d, setup = True):
138 urldata = {}
139 fn = bb.data.getVar('FILE', d, 1)
140 if fn in urldata_cache:
141 urldata = urldata_cache[fn]
142
143 for url in urls:
144 if url not in urldata:
145 urldata[url] = FetchData(url, d)
146
147 if setup:
148 for url in urldata:
149 if not urldata[url].setup:
150 urldata[url].setup_localpath(d)
151
152 urldata_cache[fn] = urldata
153 return urldata
154
155def go(d, urls = None):
156 """
157 Fetch all urls
158 init must have previously been called
159 """
160 if not urls:
161 urls = d.getVar("SRC_URI", 1).split()
162 urldata = init(urls, d, True)
163
164 for u in urls:
165 ud = urldata[u]
166 m = ud.method
167 if ud.localfile:
168 if not m.forcefetch(u, ud, d) and os.path.exists(ud.md5):
169 # File already present along with md5 stamp file
170 # Touch md5 file to show activity
171 try:
172 os.utime(ud.md5, None)
173 except:
174 # Errors aren't fatal here
175 pass
176 continue
177 lf = bb.utils.lockfile(ud.lockfile)
178 if not m.forcefetch(u, ud, d) and os.path.exists(ud.md5):
179 # If someone else fetched this before we got the lock,
180 # notice and don't try again
181 try:
182 os.utime(ud.md5, None)
183 except:
184 # Errors aren't fatal here
185 pass
186 bb.utils.unlockfile(lf)
187 continue
188 m.go(u, ud, d)
189 if ud.localfile:
190 if not m.forcefetch(u, ud, d):
191 Fetch.write_md5sum(u, ud, d)
192 bb.utils.unlockfile(lf)
193
194
195def checkstatus(d):
196 """
197 Check all urls exist upstream
198 init must have previously been called
199 """
200 urldata = init([], d, True)
201
202 for u in urldata:
203 ud = urldata[u]
204 m = ud.method
205 bb.msg.note(1, bb.msg.domain.Fetcher, "Testing URL %s" % u)
206 ret = m.checkstatus(u, ud, d)
207 if not ret:
208 bb.msg.fatal(bb.msg.domain.Fetcher, "URL %s doesn't work" % u)
209
210def localpaths(d):
211 """
212 Return a list of the local filenames, assuming successful fetch
213 """
214 local = []
215 urldata = init([], d, True)
216
217 for u in urldata:
218 ud = urldata[u]
219 local.append(ud.localpath)
220
221 return local
222
223srcrev_internal_call = False
224
225def get_srcrev(d):
226 """
227 Return the version string for the current package
228 (usually to be used as PV)
229 Most packages usually only have one SCM so we just pass on the call.
230 In the multi SCM case, we build a value based on SRCREV_FORMAT which must
231 have been set.
232 """
233
234 #
235 # Ugly code alert. localpath in the fetchers will try to evaluate SRCREV which
236 # could translate into a call to here. If it does, we need to catch this
237 # and provide some way so it knows get_srcrev is active instead of being
238 # some number etc. hence the srcrev_internal_call tracking and the magic
239 # "SRCREVINACTION" return value.
240 #
241 # Neater solutions welcome!
242 #
243 if bb.fetch.srcrev_internal_call:
244 return "SRCREVINACTION"
245
246 scms = []
247
248 # Only call setup_localpath on URIs which suppports_srcrev()
249 urldata = init(bb.data.getVar('SRC_URI', d, 1).split(), d, False)
250 for u in urldata:
251 ud = urldata[u]
252 if ud.method.suppports_srcrev():
253 if not ud.setup:
254 ud.setup_localpath(d)
255 scms.append(u)
256
257 if len(scms) == 0:
258 bb.msg.error(bb.msg.domain.Fetcher, "SRCREV was used yet no valid SCM was found in SRC_URI")
259 raise ParameterError
260
261 bb.data.setVar('__BB_DONT_CACHE','1', d)
262
263 if len(scms) == 1:
264 return urldata[scms[0]].method.sortable_revision(scms[0], urldata[scms[0]], d)
265
266 #
267 # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT
268 #
269 format = bb.data.getVar('SRCREV_FORMAT', d, 1)
270 if not format:
271 bb.msg.error(bb.msg.domain.Fetcher, "The SRCREV_FORMAT variable must be set when multiple SCMs are used.")
272 raise ParameterError
273
274 for scm in scms:
275 if 'name' in urldata[scm].parm:
276 name = urldata[scm].parm["name"]
277 rev = urldata[scm].method.sortable_revision(scm, urldata[scm], d)
278 format = format.replace(name, rev)
279
280 return format
281
282def localpath(url, d, cache = True):
283 """
284 Called from the parser with cache=False since the cache isn't ready
285 at this point. Also called from classed in OE e.g. patch.bbclass
286 """
287 ud = init([url], d)
288 if ud[url].method:
289 return ud[url].localpath
290 return url
291
292def runfetchcmd(cmd, d, quiet = False):
293 """
294 Run cmd returning the command output
295 Raise an error if interrupted or cmd fails
296 Optionally echo command output to stdout
297 """
298
299 # Need to export PATH as binary could be in metadata paths
300 # rather than host provided
301 # Also include some other variables.
302 # FIXME: Should really include all export varaiables?
303 exportvars = ['PATH', 'GIT_PROXY_COMMAND', 'GIT_PROXY_HOST', 'GIT_PROXY_PORT', 'GIT_CONFIG', 'http_proxy', 'ftp_proxy', 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', 'HOME']
304
305 for var in exportvars:
306 val = data.getVar(var, d, True)
307 if val:
308 cmd = 'export ' + var + '=%s; %s' % (val, cmd)
309
310 bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % cmd)
311
312 # redirect stderr to stdout
313 stdout_handle = os.popen(cmd + " 2>&1", "r")
314 output = ""
315
316 while 1:
317 line = stdout_handle.readline()
318 if not line:
319 break
320 if not quiet:
321 print line,
322 output += line
323
324 status = stdout_handle.close() or 0
325 signal = status >> 8
326 exitstatus = status & 0xff
327
328 if signal:
329 raise FetchError("Fetch command %s failed with signal %s, output:\n%s" % (cmd, signal, output))
330 elif status != 0:
331 raise FetchError("Fetch command %s failed with exit code %s, output:\n%s" % (cmd, status, output))
332
333 return output
334
335class FetchData(object):
336 """
337 A class which represents the fetcher state for a given URI.
338 """
339 def __init__(self, url, d):
340 self.localfile = ""
341 (self.type, self.host, self.path, self.user, self.pswd, self.parm) = bb.decodeurl(data.expand(url, d))
342 self.date = Fetch.getSRCDate(self, d)
343 self.url = url
344 if not self.user and "user" in self.parm:
345 self.user = self.parm["user"]
346 if not self.pswd and "pswd" in self.parm:
347 self.pswd = self.parm["pswd"]
348 self.setup = False
349 for m in methods:
350 if m.supports(url, self, d):
351 self.method = m
352 return
353 raise NoMethodError("Missing implementation for url %s" % url)
354
355 def setup_localpath(self, d):
356 self.setup = True
357 if "localpath" in self.parm:
358 # if user sets localpath for file, use it instead.
359 self.localpath = self.parm["localpath"]
360 else:
361 try:
362 bb.fetch.srcrev_internal_call = True
363 self.localpath = self.method.localpath(self.url, self, d)
364 finally:
365 bb.fetch.srcrev_internal_call = False
366 # We have to clear data's internal caches since the cached value of SRCREV is now wrong.
367 # Horrible...
368 bb.data.delVar("ISHOULDNEVEREXIST", d)
369 self.md5 = self.localpath + '.md5'
370 self.lockfile = self.localpath + '.lock'
371
372
373class Fetch(object):
374 """Base class for 'fetch'ing data"""
375
376 def __init__(self, urls = []):
377 self.urls = []
378
379 def supports(self, url, urldata, d):
380 """
381 Check to see if this fetch class supports a given url.
382 """
383 return 0
384
385 def localpath(self, url, urldata, d):
386 """
387 Return the local filename of a given url assuming a successful fetch.
388 Can also setup variables in urldata for use in go (saving code duplication
389 and duplicate code execution)
390 """
391 return url
392
393 def setUrls(self, urls):
394 self.__urls = urls
395
396 def getUrls(self):
397 return self.__urls
398
399 urls = property(getUrls, setUrls, None, "Urls property")
400
401 def forcefetch(self, url, urldata, d):
402 """
403 Force a fetch, even if localpath exists?
404 """
405 return False
406
407 def suppports_srcrev(self):
408 """
409 The fetcher supports auto source revisions (SRCREV)
410 """
411 return False
412
413 def go(self, url, urldata, d):
414 """
415 Fetch urls
416 Assumes localpath was called first
417 """
418 raise NoMethodError("Missing implementation for url")
419
420 def checkstatus(self, url, urldata, d):
421 """
422 Check the status of a URL
423 Assumes localpath was called first
424 """
425 bb.msg.note(1, bb.msg.domain.Fetcher, "URL %s could not be checked for status since no method exists." % url)
426 return True
427
428 def getSRCDate(urldata, d):
429 """
430 Return the SRC Date for the component
431
432 d the bb.data module
433 """
434 if "srcdate" in urldata.parm:
435 return urldata.parm['srcdate']
436
437 pn = data.getVar("PN", d, 1)
438
439 if pn:
440 return data.getVar("SRCDATE_%s" % pn, d, 1) or data.getVar("CVSDATE_%s" % pn, d, 1) or data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
441
442 return data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
443 getSRCDate = staticmethod(getSRCDate)
444
445 def srcrev_internal_helper(ud, d):
446 """
447 Return:
448 a) a source revision if specified
449 b) True if auto srcrev is in action
450 c) False otherwise
451 """
452
453 if 'rev' in ud.parm:
454 return ud.parm['rev']
455
456 if 'tag' in ud.parm:
457 return ud.parm['tag']
458
459 rev = None
460 if 'name' in ud.parm:
461 pn = data.getVar("PN", d, 1)
462 rev = data.getVar("SRCREV_pn-" + pn + "_" + ud.parm['name'], d, 1)
463 if not rev:
464 rev = data.getVar("SRCREV", d, 1)
465 if rev == "INVALID":
466 raise InvalidSRCREV("Please set SRCREV to a valid value")
467 if not rev:
468 return False
469 if rev is "SRCREVINACTION":
470 return True
471 return rev
472
473 srcrev_internal_helper = staticmethod(srcrev_internal_helper)
474
475 def localcount_internal_helper(ud, d):
476 """
477 Return:
478 a) a locked localcount if specified
479 b) None otherwise
480 """
481
482 localcount= None
483 if 'name' in ud.parm:
484 pn = data.getVar("PN", d, 1)
485 localcount = data.getVar("LOCALCOUNT_" + ud.parm['name'], d, 1)
486 if not localcount:
487 localcount = data.getVar("LOCALCOUNT", d, 1)
488 return localcount
489
490 localcount_internal_helper = staticmethod(localcount_internal_helper)
491
492 def try_mirror(d, tarfn):
493 """
494 Try to use a mirrored version of the sources. We do this
495 to avoid massive loads on foreign cvs and svn servers.
496 This method will be used by the different fetcher
497 implementations.
498
499 d Is a bb.data instance
500 tarfn is the name of the tarball
501 """
502 tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn)
503 if os.access(tarpath, os.R_OK):
504 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % tarfn)
505 return True
506
507 pn = data.getVar('PN', d, True)
508 src_tarball_stash = None
509 if pn:
510 src_tarball_stash = (data.getVar('SRC_TARBALL_STASH_%s' % pn, d, True) or data.getVar('CVS_TARBALL_STASH_%s' % pn, d, True) or data.getVar('SRC_TARBALL_STASH', d, True) or data.getVar('CVS_TARBALL_STASH', d, True) or "").split()
511
512 ld = d.createCopy()
513 for stash in src_tarball_stash:
514 url = stash + tarfn
515 try:
516 ud = FetchData(url, ld)
517 except bb.fetch.NoMethodError:
518 bb.msg.debug(1, bb.msg.domain.Fetcher, "No method for %s" % url)
519 continue
520
521 ud.setup_localpath(ld)
522
523 try:
524 ud.method.go(url, ud, ld)
525 return True
526 except (bb.fetch.MissingParameterError,
527 bb.fetch.FetchError,
528 bb.fetch.MD5SumError):
529 import sys
530 (type, value, traceback) = sys.exc_info()
531 bb.msg.debug(2, bb.msg.domain.Fetcher, "Tarball stash fetch failure: %s" % value)
532 return False
533 try_mirror = staticmethod(try_mirror)
534
535 def verify_md5sum(ud, got_sum):
536 """
537 Verify the md5sum we wanted with the one we got
538 """
539 wanted_sum = None
540 if 'md5sum' in ud.parm:
541 wanted_sum = ud.parm['md5sum']
542 if not wanted_sum:
543 return True
544
545 return wanted_sum == got_sum
546 verify_md5sum = staticmethod(verify_md5sum)
547
548 def write_md5sum(url, ud, d):
549 md5data = bb.utils.md5_file(ud.localpath)
550 # verify the md5sum
551 if not Fetch.verify_md5sum(ud, md5data):
552 raise MD5SumError(url)
553
554 md5out = file(ud.md5, 'w')
555 md5out.write(md5data)
556 md5out.close()
557 write_md5sum = staticmethod(write_md5sum)
558
559 def latest_revision(self, url, ud, d):
560 """
561 Look in the cache for the latest revision, if not present ask the SCM.
562 """
563 if not hasattr(self, "_latest_revision"):
564 raise ParameterError
565
566 pd = persist_data.PersistData(d)
567 key = self.generate_revision_key(url, ud, d)
568 rev = pd.getValue("BB_URI_HEADREVS", key)
569 if rev != None:
570 return str(rev)
571
572 rev = self._latest_revision(url, ud, d)
573 pd.setValue("BB_URI_HEADREVS", key, rev)
574 return rev
575
576 def sortable_revision(self, url, ud, d):
577 """
578
579 """
580 if hasattr(self, "_sortable_revision"):
581 return self._sortable_revision(url, ud, d)
582
583 pd = persist_data.PersistData(d)
584 key = self.generate_revision_key(url, ud, d)
585
586 latest_rev = self._build_revision(url, ud, d)
587 last_rev = pd.getValue("BB_URI_LOCALCOUNT", key + "_rev")
588 uselocalcount = bb.data.getVar("BB_LOCALCOUNT_OVERRIDE", d, True) or False
589 count = None
590 if uselocalcount:
591 count = Fetch.localcount_internal_helper(ud, d)
592 if count is None:
593 count = pd.getValue("BB_URI_LOCALCOUNT", key + "_count")
594
595 if last_rev == latest_rev:
596 return str(count + "+" + latest_rev)
597
598 buildindex_provided = hasattr(self, "_sortable_buildindex")
599 if buildindex_provided:
600 count = self._sortable_buildindex(url, ud, d, latest_rev)
601
602 if count is None:
603 count = "0"
604 elif uselocalcount or buildindex_provided:
605 count = str(count)
606 else:
607 count = str(int(count) + 1)
608
609 pd.setValue("BB_URI_LOCALCOUNT", key + "_rev", latest_rev)
610 pd.setValue("BB_URI_LOCALCOUNT", key + "_count", count)
611
612 return str(count + "+" + latest_rev)
613
614 def generate_revision_key(self, url, ud, d):
615 key = self._revision_key(url, ud, d)
616 return "%s-%s" % (key, bb.data.getVar("PN", d, True) or "")
617
618import cvs
619import git
620import local
621import svn
622import wget
623import svk
624import ssh
625import perforce
626import bzr
627import hg
628import osc
629
630methods.append(local.Local())
631methods.append(wget.Wget())
632methods.append(svn.Svn())
633methods.append(git.Git())
634methods.append(cvs.Cvs())
635methods.append(svk.Svk())
636methods.append(ssh.SSH())
637methods.append(perforce.Perforce())
638methods.append(bzr.Bzr())
639methods.append(hg.Hg())
640methods.append(osc.Osc())
diff --git a/bitbake-dev/lib/bb/fetch/bzr.py b/bitbake-dev/lib/bb/fetch/bzr.py
deleted file mode 100644
index b27fb63d07..0000000000
--- a/bitbake-dev/lib/bb/fetch/bzr.py
+++ /dev/null
@@ -1,153 +0,0 @@
1"""
2BitBake 'Fetch' implementation for bzr.
3
4"""
5
6# Copyright (C) 2007 Ross Burton
7# Copyright (C) 2007 Richard Purdie
8#
9# Classes for obtaining upstream sources for the
10# BitBake build tools.
11# Copyright (C) 2003, 2004 Chris Larson
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License version 2 as
15# published by the Free Software Foundation.
16#
17# This program is distributed in the hope that it will be useful,
18# but WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20# GNU General Public License for more details.
21#
22# You should have received a copy of the GNU General Public License along
23# with this program; if not, write to the Free Software Foundation, Inc.,
24# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25
26import os
27import sys
28import bb
29from bb import data
30from bb.fetch import Fetch
31from bb.fetch import FetchError
32from bb.fetch import runfetchcmd
33
34class Bzr(Fetch):
35 def supports(self, url, ud, d):
36 return ud.type in ['bzr']
37
38 def localpath (self, url, ud, d):
39
40 # Create paths to bzr checkouts
41 relpath = ud.path
42 if relpath.startswith('/'):
43 # Remove leading slash as os.path.join can't cope
44 relpath = relpath[1:]
45 ud.pkgdir = os.path.join(data.expand('${BZRDIR}', d), ud.host, relpath)
46
47 revision = Fetch.srcrev_internal_helper(ud, d)
48 if revision is True:
49 ud.revision = self.latest_revision(url, ud, d)
50 elif revision:
51 ud.revision = revision
52
53 if not ud.revision:
54 ud.revision = self.latest_revision(url, ud, d)
55
56 ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d)
57
58 return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
59
60 def _buildbzrcommand(self, ud, d, command):
61 """
62 Build up an bzr commandline based on ud
63 command is "fetch", "update", "revno"
64 """
65
66 basecmd = data.expand('${FETCHCMD_bzr}', d)
67
68 proto = "http"
69 if "proto" in ud.parm:
70 proto = ud.parm["proto"]
71
72 bzrroot = ud.host + ud.path
73
74 options = []
75
76 if command is "revno":
77 bzrcmd = "%s revno %s %s://%s" % (basecmd, " ".join(options), proto, bzrroot)
78 else:
79 if ud.revision:
80 options.append("-r %s" % ud.revision)
81
82 if command is "fetch":
83 bzrcmd = "%s co %s %s://%s" % (basecmd, " ".join(options), proto, bzrroot)
84 elif command is "update":
85 bzrcmd = "%s pull %s --overwrite" % (basecmd, " ".join(options))
86 else:
87 raise FetchError("Invalid bzr command %s" % command)
88
89 return bzrcmd
90
91 def go(self, loc, ud, d):
92 """Fetch url"""
93
94 # try to use the tarball stash
95 if Fetch.try_mirror(d, ud.localfile):
96 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping bzr checkout." % ud.localpath)
97 return
98
99 if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
100 bzrcmd = self._buildbzrcommand(ud, d, "update")
101 bb.msg.debug(1, bb.msg.domain.Fetcher, "BZR Update %s" % loc)
102 os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path)))
103 runfetchcmd(bzrcmd, d)
104 else:
105 os.system("rm -rf %s" % os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)))
106 bzrcmd = self._buildbzrcommand(ud, d, "fetch")
107 bb.msg.debug(1, bb.msg.domain.Fetcher, "BZR Checkout %s" % loc)
108 bb.mkdirhier(ud.pkgdir)
109 os.chdir(ud.pkgdir)
110 bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % bzrcmd)
111 runfetchcmd(bzrcmd, d)
112
113 os.chdir(ud.pkgdir)
114 # tar them up to a defined filename
115 try:
116 runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.pkgdir)), d)
117 except:
118 t, v, tb = sys.exc_info()
119 try:
120 os.unlink(ud.localpath)
121 except OSError:
122 pass
123 raise t, v, tb
124
125 def suppports_srcrev(self):
126 return True
127
128 def _revision_key(self, url, ud, d):
129 """
130 Return a unique key for the url
131 """
132 return "bzr:" + ud.pkgdir
133
134 def _latest_revision(self, url, ud, d):
135 """
136 Return the latest upstream revision number
137 """
138 bb.msg.debug(2, bb.msg.domain.Fetcher, "BZR fetcher hitting network for %s" % url)
139
140 output = runfetchcmd(self._buildbzrcommand(ud, d, "revno"), d, True)
141
142 return output.strip()
143
144 def _sortable_revision(self, url, ud, d):
145 """
146 Return a sortable revision number which in our case is the revision number
147 """
148
149 return self._build_revision(url, ud, d)
150
151 def _build_revision(self, url, ud, d):
152 return ud.revision
153
diff --git a/bitbake-dev/lib/bb/fetch/cvs.py b/bitbake-dev/lib/bb/fetch/cvs.py
deleted file mode 100644
index 90a006500e..0000000000
--- a/bitbake-dev/lib/bb/fetch/cvs.py
+++ /dev/null
@@ -1,182 +0,0 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' implementations
5
6Classes for obtaining upstream sources for the
7BitBake build tools.
8
9"""
10
11# Copyright (C) 2003, 2004 Chris Larson
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License version 2 as
15# published by the Free Software Foundation.
16#
17# This program is distributed in the hope that it will be useful,
18# but WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20# GNU General Public License for more details.
21#
22# You should have received a copy of the GNU General Public License along
23# with this program; if not, write to the Free Software Foundation, Inc.,
24# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25#
26#Based on functions from the base bb module, Copyright 2003 Holger Schurig
27#
28
29import os
30import bb
31from bb import data
32from bb.fetch import Fetch
33from bb.fetch import FetchError
34from bb.fetch import MissingParameterError
35
36class Cvs(Fetch):
37 """
38 Class to fetch a module or modules from cvs repositories
39 """
40 def supports(self, url, ud, d):
41 """
42 Check to see if a given url can be fetched with cvs.
43 """
44 return ud.type in ['cvs']
45
46 def localpath(self, url, ud, d):
47 if not "module" in ud.parm:
48 raise MissingParameterError("cvs method needs a 'module' parameter")
49 ud.module = ud.parm["module"]
50
51 ud.tag = ""
52 if 'tag' in ud.parm:
53 ud.tag = ud.parm['tag']
54
55 # Override the default date in certain cases
56 if 'date' in ud.parm:
57 ud.date = ud.parm['date']
58 elif ud.tag:
59 ud.date = ""
60
61 norecurse = ''
62 if 'norecurse' in ud.parm:
63 norecurse = '_norecurse'
64
65 fullpath = ''
66 if 'fullpath' in ud.parm:
67 fullpath = '_fullpath'
68
69 ud.localfile = data.expand('%s_%s_%s_%s%s%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date, norecurse, fullpath), d)
70
71 return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
72
73 def forcefetch(self, url, ud, d):
74 if (ud.date == "now"):
75 return True
76 return False
77
78 def go(self, loc, ud, d):
79
80 # try to use the tarball stash
81 if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
82 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping cvs checkout." % ud.localpath)
83 return
84
85 method = "pserver"
86 if "method" in ud.parm:
87 method = ud.parm["method"]
88
89 localdir = ud.module
90 if "localdir" in ud.parm:
91 localdir = ud.parm["localdir"]
92
93 cvs_port = ""
94 if "port" in ud.parm:
95 cvs_port = ud.parm["port"]
96
97 cvs_rsh = None
98 if method == "ext":
99 if "rsh" in ud.parm:
100 cvs_rsh = ud.parm["rsh"]
101
102 if method == "dir":
103 cvsroot = ud.path
104 else:
105 cvsroot = ":" + method
106 cvsproxyhost = data.getVar('CVS_PROXY_HOST', d, True)
107 if cvsproxyhost:
108 cvsroot += ";proxy=" + cvsproxyhost
109 cvsproxyport = data.getVar('CVS_PROXY_PORT', d, True)
110 if cvsproxyport:
111 cvsroot += ";proxyport=" + cvsproxyport
112 cvsroot += ":" + ud.user
113 if ud.pswd:
114 cvsroot += ":" + ud.pswd
115 cvsroot += "@" + ud.host + ":" + cvs_port + ud.path
116
117 options = []
118 if 'norecurse' in ud.parm:
119 options.append("-l")
120 if ud.date:
121 # treat YYYYMMDDHHMM specially for CVS
122 if len(ud.date) == 12:
123 options.append("-D \"%s %s:%s UTC\"" % (ud.date[0:8], ud.date[8:10], ud.date[10:12]))
124 else:
125 options.append("-D \"%s UTC\"" % ud.date)
126 if ud.tag:
127 options.append("-r %s" % ud.tag)
128
129 localdata = data.createCopy(d)
130 data.setVar('OVERRIDES', "cvs:%s" % data.getVar('OVERRIDES', localdata), localdata)
131 data.update_data(localdata)
132
133 data.setVar('CVSROOT', cvsroot, localdata)
134 data.setVar('CVSCOOPTS', " ".join(options), localdata)
135 data.setVar('CVSMODULE', ud.module, localdata)
136 cvscmd = data.getVar('FETCHCOMMAND', localdata, 1)
137 cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, 1)
138
139 if cvs_rsh:
140 cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd)
141 cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd)
142
143 # create module directory
144 bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory")
145 pkg = data.expand('${PN}', d)
146 pkgdir = os.path.join(data.expand('${CVSDIR}', localdata), pkg)
147 moddir = os.path.join(pkgdir,localdir)
148 if os.access(os.path.join(moddir,'CVS'), os.R_OK):
149 bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc)
150 # update sources there
151 os.chdir(moddir)
152 myret = os.system(cvsupdatecmd)
153 else:
154 bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
155 # check out sources there
156 bb.mkdirhier(pkgdir)
157 os.chdir(pkgdir)
158 bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % cvscmd)
159 myret = os.system(cvscmd)
160
161 if myret != 0 or not os.access(moddir, os.R_OK):
162 try:
163 os.rmdir(moddir)
164 except OSError:
165 pass
166 raise FetchError(ud.module)
167
168 # tar them up to a defined filename
169 if 'fullpath' in ud.parm:
170 os.chdir(pkgdir)
171 myret = os.system("tar -czf %s %s" % (ud.localpath, localdir))
172 else:
173 os.chdir(moddir)
174 os.chdir('..')
175 myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(moddir)))
176
177 if myret != 0:
178 try:
179 os.unlink(ud.localpath)
180 except OSError:
181 pass
182 raise FetchError(ud.module)
diff --git a/bitbake-dev/lib/bb/fetch/git.py b/bitbake-dev/lib/bb/fetch/git.py
deleted file mode 100644
index 0e68325db9..0000000000
--- a/bitbake-dev/lib/bb/fetch/git.py
+++ /dev/null
@@ -1,216 +0,0 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' git implementation
5
6"""
7
8#Copyright (C) 2005 Richard Purdie
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License version 2 as
12# published by the Free Software Foundation.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License along
20# with this program; if not, write to the Free Software Foundation, Inc.,
21# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22
23import os
24import bb
25from bb import data
26from bb.fetch import Fetch
27from bb.fetch import runfetchcmd
28
29class Git(Fetch):
30 """Class to fetch a module or modules from git repositories"""
31 def init(self, d):
32 #
33 # Only enable _sortable revision if the key is set
34 #
35 if bb.data.getVar("BB_GIT_CLONE_FOR_SRCREV", d, True):
36 self._sortable_buildindex = self._sortable_buildindex_disabled
37 def supports(self, url, ud, d):
38 """
39 Check to see if a given url can be fetched with git.
40 """
41 return ud.type in ['git']
42
43 def localpath(self, url, ud, d):
44
45 if 'protocol' in ud.parm:
46 ud.proto = ud.parm['protocol']
47 elif not ud.host:
48 ud.proto = 'file'
49 else:
50 ud.proto = "rsync"
51
52 ud.branch = ud.parm.get("branch", "master")
53
54 gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.'))
55 ud.mirrortarball = 'git_%s.tar.gz' % (gitsrcname)
56 ud.clonedir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
57
58 tag = Fetch.srcrev_internal_helper(ud, d)
59 if tag is True:
60 ud.tag = self.latest_revision(url, ud, d)
61 elif tag:
62 ud.tag = tag
63
64 if not ud.tag or ud.tag == "master":
65 ud.tag = self.latest_revision(url, ud, d)
66
67 subdir = ud.parm.get("subpath", "")
68 if subdir != "":
69 if subdir.endswith("/"):
70 subdir = subdir[:-1]
71 subdirpath = os.path.join(ud.path, subdir);
72 else:
73 subdirpath = ud.path;
74
75 if 'fullclone' in ud.parm:
76 ud.localfile = ud.mirrortarball
77 else:
78 ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, subdirpath.replace('/', '.'), ud.tag), d)
79
80 return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
81
82 def go(self, loc, ud, d):
83 """Fetch url"""
84
85 if Fetch.try_mirror(d, ud.localfile):
86 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath)
87 return
88
89 if ud.user:
90 username = ud.user + '@'
91 else:
92 username = ""
93
94 repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
95
96 coname = '%s' % (ud.tag)
97 codir = os.path.join(ud.clonedir, coname)
98
99 if not os.path.exists(ud.clonedir):
100 if Fetch.try_mirror(d, ud.mirrortarball):
101 bb.mkdirhier(ud.clonedir)
102 os.chdir(ud.clonedir)
103 runfetchcmd("tar -xzf %s" % (repofile), d)
104 else:
105 runfetchcmd("git clone -n %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.clonedir), d)
106
107 os.chdir(ud.clonedir)
108 # Remove all but the .git directory
109 if not self._contains_ref(ud.tag, d):
110 runfetchcmd("rm * -Rf", d)
111 runfetchcmd("git fetch %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.branch), d)
112 runfetchcmd("git fetch --tags %s://%s%s%s" % (ud.proto, username, ud.host, ud.path), d)
113 runfetchcmd("git prune-packed", d)
114 runfetchcmd("git pack-redundant --all | xargs -r rm", d)
115
116 os.chdir(ud.clonedir)
117 mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
118 if mirror_tarballs != "0" or 'fullclone' in ud.parm:
119 bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository")
120 runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
121
122 if 'fullclone' in ud.parm:
123 return
124
125 if os.path.exists(codir):
126 bb.utils.prunedir(codir)
127
128 subdir = ud.parm.get("subpath", "")
129 if subdir != "":
130 if subdir.endswith("/"):
131 subdirbase = os.path.basename(subdir[:-1])
132 else:
133 subdirbase = os.path.basename(subdir)
134 else:
135 subdirbase = ""
136
137 if subdir != "":
138 readpathspec = ":%s" % (subdir)
139 codir = os.path.join(codir, "git")
140 coprefix = os.path.join(codir, subdirbase, "")
141 else:
142 readpathspec = ""
143 coprefix = os.path.join(codir, "git", "")
144
145 bb.mkdirhier(codir)
146 os.chdir(ud.clonedir)
147 runfetchcmd("git read-tree %s%s" % (ud.tag, readpathspec), d)
148 runfetchcmd("git checkout-index -q -f --prefix=%s -a" % (coprefix), d)
149
150 os.chdir(codir)
151 bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout")
152 runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
153
154 os.chdir(ud.clonedir)
155 bb.utils.prunedir(codir)
156
157 def suppports_srcrev(self):
158 return True
159
160 def _contains_ref(self, tag, d):
161 output = runfetchcmd("git log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % tag, d, quiet=True)
162 return output.split()[0] != "0"
163
164 def _revision_key(self, url, ud, d):
165 """
166 Return a unique key for the url
167 """
168 return "git:" + ud.host + ud.path.replace('/', '.')
169
170 def _latest_revision(self, url, ud, d):
171 """
172 Compute the HEAD revision for the url
173 """
174 if ud.user:
175 username = ud.user + '@'
176 else:
177 username = ""
178
179 cmd = "git ls-remote %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.branch)
180 output = runfetchcmd(cmd, d, True)
181 if not output:
182 raise bb.fetch.FetchError("Fetch command %s gave empty output\n" % (cmd))
183 return output.split()[0]
184
185 def _build_revision(self, url, ud, d):
186 return ud.tag
187
188 def _sortable_buildindex_disabled(self, url, ud, d, rev):
189 """
190 Return a suitable buildindex for the revision specified. This is done by counting revisions
191 using "git rev-list" which may or may not work in different circumstances.
192 """
193
194 cwd = os.getcwd()
195
196 # Check if we have the rev already
197
198 if not os.path.exists(ud.clonedir):
199 print "no repo"
200 self.go(None, ud, d)
201 if not os.path.exists(ud.clonedir):
202 bb.msg.error(bb.msg.domain.Fetcher, "GIT repository for %s doesn't exist in %s, cannot get sortable buildnumber, using old value" % (url, ud.clonedir))
203 return None
204
205
206 os.chdir(ud.clonedir)
207 if not self._contains_ref(rev, d):
208 self.go(None, ud, d)
209
210 output = runfetchcmd("git rev-list %s -- 2> /dev/null | wc -l" % rev, d, quiet=True)
211 os.chdir(cwd)
212
213 buildindex = "%s" % output.split()[0]
214 bb.msg.debug(1, bb.msg.domain.Fetcher, "GIT repository for %s in %s is returning %s revisions in rev-list before %s" % (url, repodir, buildindex, rev))
215 return buildindex
216
diff --git a/bitbake-dev/lib/bb/fetch/hg.py b/bitbake-dev/lib/bb/fetch/hg.py
deleted file mode 100644
index 08cb61fc28..0000000000
--- a/bitbake-dev/lib/bb/fetch/hg.py
+++ /dev/null
@@ -1,178 +0,0 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' implementation for mercurial DRCS (hg).
5
6"""
7
8# Copyright (C) 2003, 2004 Chris Larson
9# Copyright (C) 2004 Marcin Juszkiewicz
10# Copyright (C) 2007 Robert Schuster
11#
12# This program is free software; you can redistribute it and/or modify
13# it under the terms of the GNU General Public License version 2 as
14# published by the Free Software Foundation.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License along
22# with this program; if not, write to the Free Software Foundation, Inc.,
23# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24#
25# Based on functions from the base bb module, Copyright 2003 Holger Schurig
26
27import os
28import sys
29import bb
30from bb import data
31from bb.fetch import Fetch
32from bb.fetch import FetchError
33from bb.fetch import MissingParameterError
34from bb.fetch import runfetchcmd
35
36class Hg(Fetch):
37 """Class to fetch a from mercurial repositories"""
38 def supports(self, url, ud, d):
39 """
40 Check to see if a given url can be fetched with mercurial.
41 """
42 return ud.type in ['hg']
43
44 def localpath(self, url, ud, d):
45 if not "module" in ud.parm:
46 raise MissingParameterError("hg method needs a 'module' parameter")
47
48 ud.module = ud.parm["module"]
49
50 # Create paths to mercurial checkouts
51 relpath = ud.path
52 if relpath.startswith('/'):
53 # Remove leading slash as os.path.join can't cope
54 relpath = relpath[1:]
55 ud.pkgdir = os.path.join(data.expand('${HGDIR}', d), ud.host, relpath)
56 ud.moddir = os.path.join(ud.pkgdir, ud.module)
57
58 if 'rev' in ud.parm:
59 ud.revision = ud.parm['rev']
60 else:
61 tag = Fetch.srcrev_internal_helper(ud, d)
62 if tag is True:
63 ud.revision = self.latest_revision(url, ud, d)
64 elif tag:
65 ud.revision = tag
66 else:
67 ud.revision = self.latest_revision(url, ud, d)
68
69 ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d)
70
71 return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
72
73 def _buildhgcommand(self, ud, d, command):
74 """
75 Build up an hg commandline based on ud
76 command is "fetch", "update", "info"
77 """
78
79 basecmd = data.expand('${FETCHCMD_hg}', d)
80
81 proto = "http"
82 if "proto" in ud.parm:
83 proto = ud.parm["proto"]
84
85 host = ud.host
86 if proto == "file":
87 host = "/"
88 ud.host = "localhost"
89
90 if not ud.user:
91 hgroot = host + ud.path
92 else:
93 hgroot = ud.user + "@" + host + ud.path
94
95 if command is "info":
96 return "%s identify -i %s://%s/%s" % (basecmd, proto, hgroot, ud.module)
97
98 options = [];
99 if ud.revision:
100 options.append("-r %s" % ud.revision)
101
102 if command is "fetch":
103 cmd = "%s clone %s %s://%s/%s %s" % (basecmd, " ".join(options), proto, hgroot, ud.module, ud.module)
104 elif command is "pull":
105 # do not pass options list; limiting pull to rev causes the local
106 # repo not to contain it and immediately following "update" command
107 # will crash
108 cmd = "%s pull" % (basecmd)
109 elif command is "update":
110 cmd = "%s update -C %s" % (basecmd, " ".join(options))
111 else:
112 raise FetchError("Invalid hg command %s" % command)
113
114 return cmd
115
116 def go(self, loc, ud, d):
117 """Fetch url"""
118
119 # try to use the tarball stash
120 if Fetch.try_mirror(d, ud.localfile):
121 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping hg checkout." % ud.localpath)
122 return
123
124 bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
125
126 if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
127 updatecmd = self._buildhgcommand(ud, d, "pull")
128 bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc)
129 # update sources there
130 os.chdir(ud.moddir)
131 bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % updatecmd)
132 runfetchcmd(updatecmd, d)
133
134 else:
135 fetchcmd = self._buildhgcommand(ud, d, "fetch")
136 bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
137 # check out sources there
138 bb.mkdirhier(ud.pkgdir)
139 os.chdir(ud.pkgdir)
140 bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % fetchcmd)
141 runfetchcmd(fetchcmd, d)
142
143 # Even when we clone (fetch), we still need to update as hg's clone
144 # won't checkout the specified revision if its on a branch
145 updatecmd = self._buildhgcommand(ud, d, "update")
146 bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % updatecmd)
147 runfetchcmd(updatecmd, d)
148
149 os.chdir(ud.pkgdir)
150 try:
151 runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d)
152 except:
153 t, v, tb = sys.exc_info()
154 try:
155 os.unlink(ud.localpath)
156 except OSError:
157 pass
158 raise t, v, tb
159
160 def suppports_srcrev(self):
161 return True
162
163 def _latest_revision(self, url, ud, d):
164 """
165 Compute tip revision for the url
166 """
167 output = runfetchcmd(self._buildhgcommand(ud, d, "info"), d)
168 return output.strip()
169
170 def _build_revision(self, url, ud, d):
171 return ud.revision
172
173 def _revision_key(self, url, ud, d):
174 """
175 Return a unique key for the url
176 """
177 return "hg:" + ud.moddir
178
diff --git a/bitbake-dev/lib/bb/fetch/local.py b/bitbake-dev/lib/bb/fetch/local.py
deleted file mode 100644
index f9bdf589cb..0000000000
--- a/bitbake-dev/lib/bb/fetch/local.py
+++ /dev/null
@@ -1,72 +0,0 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' implementations
5
6Classes for obtaining upstream sources for the
7BitBake build tools.
8
9"""
10
11# Copyright (C) 2003, 2004 Chris Larson
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License version 2 as
15# published by the Free Software Foundation.
16#
17# This program is distributed in the hope that it will be useful,
18# but WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20# GNU General Public License for more details.
21#
22# You should have received a copy of the GNU General Public License along
23# with this program; if not, write to the Free Software Foundation, Inc.,
24# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25#
26# Based on functions from the base bb module, Copyright 2003 Holger Schurig
27
28import os
29import bb
30from bb import data
31from bb.fetch import Fetch
32
33class Local(Fetch):
34 def supports(self, url, urldata, d):
35 """
36 Check to see if a given url represents a local fetch.
37 """
38 return urldata.type in ['file']
39
40 def localpath(self, url, urldata, d):
41 """
42 Return the local filename of a given url assuming a successful fetch.
43 """
44 path = url.split("://")[1]
45 path = path.split(";")[0]
46 newpath = path
47 if path[0] != "/":
48 filespath = data.getVar('FILESPATH', d, 1)
49 if filespath:
50 newpath = bb.which(filespath, path)
51 if not newpath:
52 filesdir = data.getVar('FILESDIR', d, 1)
53 if filesdir:
54 newpath = os.path.join(filesdir, path)
55 # We don't set localfile as for this fetcher the file is already local!
56 return newpath
57
58 def go(self, url, urldata, d):
59 """Fetch urls (no-op for Local method)"""
60 # no need to fetch local files, we'll deal with them in place.
61 return 1
62
63 def checkstatus(self, url, urldata, d):
64 """
65 Check the status of the url
66 """
67 if urldata.localpath.find("*") != -1:
68 bb.msg.note(1, bb.msg.domain.Fetcher, "URL %s looks like a glob and was therefore not checked." % url)
69 return True
70 if os.path.exists(urldata.localpath):
71 return True
72 return False
diff --git a/bitbake-dev/lib/bb/fetch/osc.py b/bitbake-dev/lib/bb/fetch/osc.py
deleted file mode 100644
index 2c34caf6c9..0000000000
--- a/bitbake-dev/lib/bb/fetch/osc.py
+++ /dev/null
@@ -1,155 +0,0 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4Bitbake "Fetch" implementation for osc (Opensuse build service client).
5Based on the svn "Fetch" implementation.
6
7"""
8
9import os
10import sys
11import bb
12from bb import data
13from bb.fetch import Fetch
14from bb.fetch import FetchError
15from bb.fetch import MissingParameterError
16from bb.fetch import runfetchcmd
17
18class Osc(Fetch):
19 """Class to fetch a module or modules from Opensuse build server
20 repositories."""
21
22 def supports(self, url, ud, d):
23 """
24 Check to see if a given url can be fetched with osc.
25 """
26 return ud.type in ['osc']
27
28 def localpath(self, url, ud, d):
29 if not "module" in ud.parm:
30 raise MissingParameterError("osc method needs a 'module' parameter.")
31
32 ud.module = ud.parm["module"]
33
34 # Create paths to osc checkouts
35 relpath = ud.path
36 if relpath.startswith('/'):
37 # Remove leading slash as os.path.join can't cope
38 relpath = relpath[1:]
39 ud.pkgdir = os.path.join(data.expand('${OSCDIR}', d), ud.host)
40 ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module)
41
42 if 'rev' in ud.parm:
43 ud.revision = ud.parm['rev']
44 else:
45 pv = data.getVar("PV", d, 0)
46 rev = Fetch.srcrev_internal_helper(ud, d)
47 if rev and rev != True:
48 ud.revision = rev
49 else:
50 ud.revision = ""
51
52 ud.localfile = data.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision), d)
53
54 return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
55
56 def _buildosccommand(self, ud, d, command):
57 """
58 Build up an ocs commandline based on ud
59 command is "fetch", "update", "info"
60 """
61
62 basecmd = data.expand('${FETCHCMD_osc}', d)
63
64 proto = "ocs"
65 if "proto" in ud.parm:
66 proto = ud.parm["proto"]
67
68 options = []
69
70 config = "-c %s" % self.generate_config(ud, d)
71
72 if ud.revision:
73 options.append("-r %s" % ud.revision)
74
75 coroot = ud.path
76 if coroot.startswith('/'):
77 # Remove leading slash as os.path.join can't cope
78 coroot= coroot[1:]
79
80 if command is "fetch":
81 osccmd = "%s %s co %s/%s %s" % (basecmd, config, coroot, ud.module, " ".join(options))
82 elif command is "update":
83 osccmd = "%s %s up %s" % (basecmd, config, " ".join(options))
84 else:
85 raise FetchError("Invalid osc command %s" % command)
86
87 return osccmd
88
89 def go(self, loc, ud, d):
90 """
91 Fetch url
92 """
93
94 # Try to use the tarball stash
95 if Fetch.try_mirror(d, ud.localfile):
96 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping osc checkout." % ud.localpath)
97 return
98
99 bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
100
101 if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK):
102 oscupdatecmd = self._buildosccommand(ud, d, "update")
103 bb.msg.note(1, bb.msg.domain.Fetcher, "Update "+ loc)
104 # update sources there
105 os.chdir(ud.moddir)
106 bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % oscupdatecmd)
107 runfetchcmd(oscupdatecmd, d)
108 else:
109 oscfetchcmd = self._buildosccommand(ud, d, "fetch")
110 bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
111 # check out sources there
112 bb.mkdirhier(ud.pkgdir)
113 os.chdir(ud.pkgdir)
114 bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % oscfetchcmd)
115 runfetchcmd(oscfetchcmd, d)
116
117 os.chdir(os.path.join(ud.pkgdir + ud.path))
118 # tar them up to a defined filename
119 try:
120 runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d)
121 except:
122 t, v, tb = sys.exc_info()
123 try:
124 os.unlink(ud.localpath)
125 except OSError:
126 pass
127 raise t, v, tb
128
129 def supports_srcrev(self):
130 return False
131
132 def generate_config(self, ud, d):
133 """
134 Generate a .oscrc to be used for this run.
135 """
136
137 config_path = "%s/oscrc" % data.expand('${OSCDIR}', d)
138 if (os.path.exists(config_path)):
139 os.remove(config_path)
140
141 f = open(config_path, 'w')
142 f.write("[general]\n")
143 f.write("apisrv = %s\n" % ud.host)
144 f.write("scheme = http\n")
145 f.write("su-wrapper = su -c\n")
146 f.write("build-root = %s\n" % data.expand('${WORKDIR}', d))
147 f.write("urllist = http://moblin-obs.jf.intel.com:8888/build/%(project)s/%(repository)s/%(buildarch)s/:full/%(name)s.rpm\n")
148 f.write("extra-pkgs = gzip\n")
149 f.write("\n")
150 f.write("[%s]\n" % ud.host)
151 f.write("user = %s\n" % ud.parm["user"])
152 f.write("pass = %s\n" % ud.parm["pswd"])
153 f.close()
154
155 return config_path
diff --git a/bitbake-dev/lib/bb/fetch/perforce.py b/bitbake-dev/lib/bb/fetch/perforce.py
deleted file mode 100644
index 394f5a2253..0000000000
--- a/bitbake-dev/lib/bb/fetch/perforce.py
+++ /dev/null
@@ -1,214 +0,0 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' implementations
5
6Classes for obtaining upstream sources for the
7BitBake build tools.
8
9"""
10
11# Copyright (C) 2003, 2004 Chris Larson
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License version 2 as
15# published by the Free Software Foundation.
16#
17# This program is distributed in the hope that it will be useful,
18# but WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20# GNU General Public License for more details.
21#
22# You should have received a copy of the GNU General Public License along
23# with this program; if not, write to the Free Software Foundation, Inc.,
24# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25#
26# Based on functions from the base bb module, Copyright 2003 Holger Schurig
27
28import os
29import bb
30from bb import data
31from bb.fetch import Fetch
32from bb.fetch import FetchError
33
34class Perforce(Fetch):
35 def supports(self, url, ud, d):
36 return ud.type in ['p4']
37
38 def doparse(url,d):
39 parm = {}
40 path = url.split("://")[1]
41 delim = path.find("@");
42 if delim != -1:
43 (user,pswd,host,port) = path.split('@')[0].split(":")
44 path = path.split('@')[1]
45 else:
46 (host,port) = data.getVar('P4PORT', d).split(':')
47 user = ""
48 pswd = ""
49
50 if path.find(";") != -1:
51 keys=[]
52 values=[]
53 plist = path.split(';')
54 for item in plist:
55 if item.count('='):
56 (key,value) = item.split('=')
57 keys.append(key)
58 values.append(value)
59
60 parm = dict(zip(keys,values))
61 path = "//" + path.split(';')[0]
62 host += ":%s" % (port)
63 parm["cset"] = Perforce.getcset(d, path, host, user, pswd, parm)
64
65 return host,path,user,pswd,parm
66 doparse = staticmethod(doparse)
67
68 def getcset(d, depot,host,user,pswd,parm):
69 p4opt = ""
70 if "cset" in parm:
71 return parm["cset"];
72 if user:
73 p4opt += " -u %s" % (user)
74 if pswd:
75 p4opt += " -P %s" % (pswd)
76 if host:
77 p4opt += " -p %s" % (host)
78
79 p4date = data.getVar("P4DATE", d, 1)
80 if "revision" in parm:
81 depot += "#%s" % (parm["revision"])
82 elif "label" in parm:
83 depot += "@%s" % (parm["label"])
84 elif p4date:
85 depot += "@%s" % (p4date)
86
87 p4cmd = data.getVar('FETCHCOMMAND_p4', d, 1)
88 bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s%s changes -m 1 %s" % (p4cmd, p4opt, depot))
89 p4file = os.popen("%s%s changes -m 1 %s" % (p4cmd, p4opt, depot))
90 cset = p4file.readline().strip()
91 bb.msg.debug(1, bb.msg.domain.Fetcher, "READ %s" % (cset))
92 if not cset:
93 return -1
94
95 return cset.split(' ')[1]
96 getcset = staticmethod(getcset)
97
98 def localpath(self, url, ud, d):
99
100 (host,path,user,pswd,parm) = Perforce.doparse(url,d)
101
102 # If a label is specified, we use that as our filename
103
104 if "label" in parm:
105 ud.localfile = "%s.tar.gz" % (parm["label"])
106 return os.path.join(data.getVar("DL_DIR", d, 1), ud.localfile)
107
108 base = path
109 which = path.find('/...')
110 if which != -1:
111 base = path[:which]
112
113 if base[0] == "/":
114 base = base[1:]
115
116 cset = Perforce.getcset(d, path, host, user, pswd, parm)
117
118 ud.localfile = data.expand('%s+%s+%s.tar.gz' % (host,base.replace('/', '.'), cset), d)
119
120 return os.path.join(data.getVar("DL_DIR", d, 1), ud.localfile)
121
122 def go(self, loc, ud, d):
123 """
124 Fetch urls
125 """
126
127 # try to use the tarball stash
128 if Fetch.try_mirror(d, ud.localfile):
129 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping perforce checkout." % ud.localpath)
130 return
131
132 (host,depot,user,pswd,parm) = Perforce.doparse(loc, d)
133
134 if depot.find('/...') != -1:
135 path = depot[:depot.find('/...')]
136 else:
137 path = depot
138
139 if "module" in parm:
140 module = parm["module"]
141 else:
142 module = os.path.basename(path)
143
144 localdata = data.createCopy(d)
145 data.setVar('OVERRIDES', "p4:%s" % data.getVar('OVERRIDES', localdata), localdata)
146 data.update_data(localdata)
147
148 # Get the p4 command
149 p4opt = ""
150 if user:
151 p4opt += " -u %s" % (user)
152
153 if pswd:
154 p4opt += " -P %s" % (pswd)
155
156 if host:
157 p4opt += " -p %s" % (host)
158
159 p4cmd = data.getVar('FETCHCOMMAND', localdata, 1)
160
161 # create temp directory
162 bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: creating temporary directory")
163 bb.mkdirhier(data.expand('${WORKDIR}', localdata))
164 data.setVar('TMPBASE', data.expand('${WORKDIR}/oep4.XXXXXX', localdata), localdata)
165 tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
166 tmpfile = tmppipe.readline().strip()
167 if not tmpfile:
168 bb.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
169 raise FetchError(module)
170
171 if "label" in parm:
172 depot = "%s@%s" % (depot,parm["label"])
173 else:
174 cset = Perforce.getcset(d, depot, host, user, pswd, parm)
175 depot = "%s@%s" % (depot,cset)
176
177 os.chdir(tmpfile)
178 bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
179 bb.msg.note(1, bb.msg.domain.Fetcher, "%s%s files %s" % (p4cmd, p4opt, depot))
180 p4file = os.popen("%s%s files %s" % (p4cmd, p4opt, depot))
181
182 if not p4file:
183 bb.error("Fetch: unable to get the P4 files from %s" % (depot))
184 raise FetchError(module)
185
186 count = 0
187
188 for file in p4file:
189 list = file.split()
190
191 if list[2] == "delete":
192 continue
193
194 dest = list[0][len(path)+1:]
195 where = dest.find("#")
196
197 os.system("%s%s print -o %s/%s %s" % (p4cmd, p4opt, module,dest[:where],list[0]))
198 count = count + 1
199
200 if count == 0:
201 bb.error("Fetch: No files gathered from the P4 fetch")
202 raise FetchError(module)
203
204 myret = os.system("tar -czf %s %s" % (ud.localpath, module))
205 if myret != 0:
206 try:
207 os.unlink(ud.localpath)
208 except OSError:
209 pass
210 raise FetchError(module)
211 # cleanup
212 os.system('rm -rf %s' % tmpfile)
213
214
diff --git a/bitbake-dev/lib/bb/fetch/ssh.py b/bitbake-dev/lib/bb/fetch/ssh.py
deleted file mode 100644
index 68e6fdb1df..0000000000
--- a/bitbake-dev/lib/bb/fetch/ssh.py
+++ /dev/null
@@ -1,118 +0,0 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3'''
4BitBake 'Fetch' implementations
5
6This implementation is for Secure Shell (SSH), and attempts to comply with the
7IETF secsh internet draft:
8 http://tools.ietf.org/wg/secsh/draft-ietf-secsh-scp-sftp-ssh-uri/
9
10 Currently does not support the sftp parameters, as this uses scp
11 Also does not support the 'fingerprint' connection parameter.
12
13'''
14
15# Copyright (C) 2006 OpenedHand Ltd.
16#
17#
18# Based in part on svk.py:
19# Copyright (C) 2006 Holger Hans Peter Freyther
20# Based on svn.py:
21# Copyright (C) 2003, 2004 Chris Larson
22# Based on functions from the base bb module:
23# Copyright 2003 Holger Schurig
24#
25#
26# This program is free software; you can redistribute it and/or modify
27# it under the terms of the GNU General Public License version 2 as
28# published by the Free Software Foundation.
29#
30# This program is distributed in the hope that it will be useful,
31# but WITHOUT ANY WARRANTY; without even the implied warranty of
32# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
33# GNU General Public License for more details.
34#
35# You should have received a copy of the GNU General Public License along
36# with this program; if not, write to the Free Software Foundation, Inc.,
37# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
38
39import re, os
40from bb import data
41from bb.fetch import Fetch
42from bb.fetch import FetchError
43
44
45__pattern__ = re.compile(r'''
46 \s* # Skip leading whitespace
47 ssh:// # scheme
48 ( # Optional username/password block
49 (?P<user>\S+) # username
50 (:(?P<pass>\S+))? # colon followed by the password (optional)
51 )?
52 (?P<cparam>(;[^;]+)*)? # connection parameters block (optional)
53 @
54 (?P<host>\S+?) # non-greedy match of the host
55 (:(?P<port>[0-9]+))? # colon followed by the port (optional)
56 /
57 (?P<path>[^;]+) # path on the remote system, may be absolute or relative,
58 # and may include the use of '~' to reference the remote home
59 # directory
60 (?P<sparam>(;[^;]+)*)? # parameters block (optional)
61 $
62''', re.VERBOSE)
63
64class SSH(Fetch):
65 '''Class to fetch a module or modules via Secure Shell'''
66
67 def supports(self, url, urldata, d):
68 return __pattern__.match(url) != None
69
70 def localpath(self, url, urldata, d):
71 m = __pattern__.match(url)
72 path = m.group('path')
73 host = m.group('host')
74 lpath = os.path.join(data.getVar('DL_DIR', d, True), host, os.path.basename(path))
75 return lpath
76
77 def go(self, url, urldata, d):
78 dldir = data.getVar('DL_DIR', d, 1)
79
80 m = __pattern__.match(url)
81 path = m.group('path')
82 host = m.group('host')
83 port = m.group('port')
84 user = m.group('user')
85 password = m.group('pass')
86
87 ldir = os.path.join(dldir, host)
88 lpath = os.path.join(ldir, os.path.basename(path))
89
90 if not os.path.exists(ldir):
91 os.makedirs(ldir)
92
93 if port:
94 port = '-P %s' % port
95 else:
96 port = ''
97
98 if user:
99 fr = user
100 if password:
101 fr += ':%s' % password
102 fr += '@%s' % host
103 else:
104 fr = host
105 fr += ':%s' % path
106
107
108 import commands
109 cmd = 'scp -B -r %s %s %s/' % (
110 port,
111 commands.mkarg(fr),
112 commands.mkarg(ldir)
113 )
114
115 (exitstatus, output) = commands.getstatusoutput(cmd)
116 if exitstatus != 0:
117 print output
118 raise FetchError('Unable to fetch %s' % url)
diff --git a/bitbake-dev/lib/bb/fetch/svk.py b/bitbake-dev/lib/bb/fetch/svk.py
deleted file mode 100644
index 120dad9d4e..0000000000
--- a/bitbake-dev/lib/bb/fetch/svk.py
+++ /dev/null
@@ -1,109 +0,0 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' implementations
5
6This implementation is for svk. It is based on the svn implementation
7
8"""
9
10# Copyright (C) 2006 Holger Hans Peter Freyther
11# Copyright (C) 2003, 2004 Chris Larson
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License version 2 as
15# published by the Free Software Foundation.
16#
17# This program is distributed in the hope that it will be useful,
18# but WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20# GNU General Public License for more details.
21#
22# You should have received a copy of the GNU General Public License along
23# with this program; if not, write to the Free Software Foundation, Inc.,
24# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25#
26# Based on functions from the base bb module, Copyright 2003 Holger Schurig
27
28import os
29import bb
30from bb import data
31from bb.fetch import Fetch
32from bb.fetch import FetchError
33from bb.fetch import MissingParameterError
34
35class Svk(Fetch):
36 """Class to fetch a module or modules from svk repositories"""
37 def supports(self, url, ud, d):
38 """
39 Check to see if a given url can be fetched with svk.
40 """
41 return ud.type in ['svk']
42
43 def localpath(self, url, ud, d):
44 if not "module" in ud.parm:
45 raise MissingParameterError("svk method needs a 'module' parameter")
46 else:
47 ud.module = ud.parm["module"]
48
49 ud.revision = ""
50 if 'rev' in ud.parm:
51 ud.revision = ud.parm['rev']
52
53 ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d)
54
55 return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
56
57 def forcefetch(self, url, ud, d):
58 if (ud.date == "now"):
59 return True
60 return False
61
62 def go(self, loc, ud, d):
63 """Fetch urls"""
64
65 if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
66 return
67
68 svkroot = ud.host + ud.path
69
70 svkcmd = "svk co -r {%s} %s/%s" % (ud.date, svkroot, ud.module)
71
72 if ud.revision:
73 svkcmd = "svk co -r %s %s/%s" % (ud.revision, svkroot, ud.module)
74
75 # create temp directory
76 localdata = data.createCopy(d)
77 data.update_data(localdata)
78 bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: creating temporary directory")
79 bb.mkdirhier(data.expand('${WORKDIR}', localdata))
80 data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvk.XXXXXX', localdata), localdata)
81 tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
82 tmpfile = tmppipe.readline().strip()
83 if not tmpfile:
84 bb.msg.error(bb.msg.domain.Fetcher, "Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
85 raise FetchError(ud.module)
86
87 # check out sources there
88 os.chdir(tmpfile)
89 bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
90 bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svkcmd)
91 myret = os.system(svkcmd)
92 if myret != 0:
93 try:
94 os.rmdir(tmpfile)
95 except OSError:
96 pass
97 raise FetchError(ud.module)
98
99 os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module)))
100 # tar them up to a defined filename
101 myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module)))
102 if myret != 0:
103 try:
104 os.unlink(ud.localpath)
105 except OSError:
106 pass
107 raise FetchError(ud.module)
108 # cleanup
109 os.system('rm -rf %s' % tmpfile)
diff --git a/bitbake-dev/lib/bb/fetch/svn.py b/bitbake-dev/lib/bb/fetch/svn.py
deleted file mode 100644
index eef9862a84..0000000000
--- a/bitbake-dev/lib/bb/fetch/svn.py
+++ /dev/null
@@ -1,206 +0,0 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' implementation for svn.
5
6"""
7
8# Copyright (C) 2003, 2004 Chris Larson
9# Copyright (C) 2004 Marcin Juszkiewicz
10#
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License version 2 as
13# published by the Free Software Foundation.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License along
21# with this program; if not, write to the Free Software Foundation, Inc.,
22# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
23#
24# Based on functions from the base bb module, Copyright 2003 Holger Schurig
25
26import os
27import sys
28import bb
29from bb import data
30from bb.fetch import Fetch
31from bb.fetch import FetchError
32from bb.fetch import MissingParameterError
33from bb.fetch import runfetchcmd
34
35class Svn(Fetch):
36 """Class to fetch a module or modules from svn repositories"""
37 def supports(self, url, ud, d):
38 """
39 Check to see if a given url can be fetched with svn.
40 """
41 return ud.type in ['svn']
42
43 def localpath(self, url, ud, d):
44 if not "module" in ud.parm:
45 raise MissingParameterError("svn method needs a 'module' parameter")
46
47 ud.module = ud.parm["module"]
48
49 # Create paths to svn checkouts
50 relpath = ud.path
51 if relpath.startswith('/'):
52 # Remove leading slash as os.path.join can't cope
53 relpath = relpath[1:]
54 ud.pkgdir = os.path.join(data.expand('${SVNDIR}', d), ud.host, relpath)
55 ud.moddir = os.path.join(ud.pkgdir, ud.module)
56
57 if 'rev' in ud.parm:
58 ud.date = ""
59 ud.revision = ud.parm['rev']
60 elif 'date' in ud.date:
61 ud.date = ud.parm['date']
62 ud.revision = ""
63 else:
64 #
65 # ***Nasty hack***
66 # If DATE in unexpanded PV, use ud.date (which is set from SRCDATE)
67 # Should warn people to switch to SRCREV here
68 #
69 pv = data.getVar("PV", d, 0)
70 if "DATE" in pv:
71 ud.revision = ""
72 else:
73 rev = Fetch.srcrev_internal_helper(ud, d)
74 if rev is True:
75 ud.revision = self.latest_revision(url, ud, d)
76 ud.date = ""
77 elif rev:
78 ud.revision = rev
79 ud.date = ""
80 else:
81 ud.revision = ""
82
83 ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d)
84
85 return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
86
87 def _buildsvncommand(self, ud, d, command):
88 """
89 Build up an svn commandline based on ud
90 command is "fetch", "update", "info"
91 """
92
93 basecmd = data.expand('${FETCHCMD_svn}', d)
94
95 proto = "svn"
96 if "proto" in ud.parm:
97 proto = ud.parm["proto"]
98
99 svn_rsh = None
100 if proto == "svn+ssh" and "rsh" in ud.parm:
101 svn_rsh = ud.parm["rsh"]
102
103 svnroot = ud.host + ud.path
104
105 # either use the revision, or SRCDATE in braces,
106 options = []
107
108 if ud.user:
109 options.append("--username %s" % ud.user)
110
111 if ud.pswd:
112 options.append("--password %s" % ud.pswd)
113
114 if command is "info":
115 svncmd = "%s info %s %s://%s/%s/" % (basecmd, " ".join(options), proto, svnroot, ud.module)
116 else:
117 suffix = ""
118 if ud.revision:
119 options.append("-r %s" % ud.revision)
120 suffix = "@%s" % (ud.revision)
121 elif ud.date:
122 options.append("-r {%s}" % ud.date)
123
124 if command is "fetch":
125 svncmd = "%s co %s %s://%s/%s%s %s" % (basecmd, " ".join(options), proto, svnroot, ud.module, suffix, ud.module)
126 elif command is "update":
127 svncmd = "%s update %s" % (basecmd, " ".join(options))
128 else:
129 raise FetchError("Invalid svn command %s" % command)
130
131 if svn_rsh:
132 svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd)
133
134 return svncmd
135
136 def go(self, loc, ud, d):
137 """Fetch url"""
138
139 # try to use the tarball stash
140 if Fetch.try_mirror(d, ud.localfile):
141 bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping svn checkout." % ud.localpath)
142 return
143
144 bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
145
146 if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
147 svnupdatecmd = self._buildsvncommand(ud, d, "update")
148 bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc)
149 # update sources there
150 os.chdir(ud.moddir)
151 bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnupdatecmd)
152 runfetchcmd(svnupdatecmd, d)
153 else:
154 svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
155 bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
156 # check out sources there
157 bb.mkdirhier(ud.pkgdir)
158 os.chdir(ud.pkgdir)
159 bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnfetchcmd)
160 runfetchcmd(svnfetchcmd, d)
161
162 os.chdir(ud.pkgdir)
163 # tar them up to a defined filename
164 try:
165 runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d)
166 except:
167 t, v, tb = sys.exc_info()
168 try:
169 os.unlink(ud.localpath)
170 except OSError:
171 pass
172 raise t, v, tb
173
174 def suppports_srcrev(self):
175 return True
176
177 def _revision_key(self, url, ud, d):
178 """
179 Return a unique key for the url
180 """
181 return "svn:" + ud.moddir
182
183 def _latest_revision(self, url, ud, d):
184 """
185 Return the latest upstream revision number
186 """
187 bb.msg.debug(2, bb.msg.domain.Fetcher, "SVN fetcher hitting network for %s" % url)
188
189 output = runfetchcmd("LANG=C LC_ALL=C " + self._buildsvncommand(ud, d, "info"), d, True)
190
191 revision = None
192 for line in output.splitlines():
193 if "Last Changed Rev" in line:
194 revision = line.split(":")[1].strip()
195
196 return revision
197
198 def _sortable_revision(self, url, ud, d):
199 """
200 Return a sortable revision number which in our case is the revision number
201 """
202
203 return self._build_revision(url, ud, d)
204
205 def _build_revision(self, url, ud, d):
206 return ud.revision
diff --git a/bitbake-dev/lib/bb/fetch/wget.py b/bitbake-dev/lib/bb/fetch/wget.py
deleted file mode 100644
index fd93c7ec46..0000000000
--- a/bitbake-dev/lib/bb/fetch/wget.py
+++ /dev/null
@@ -1,130 +0,0 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' implementations
5
6Classes for obtaining upstream sources for the
7BitBake build tools.
8
9"""
10
11# Copyright (C) 2003, 2004 Chris Larson
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License version 2 as
15# published by the Free Software Foundation.
16#
17# This program is distributed in the hope that it will be useful,
18# but WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20# GNU General Public License for more details.
21#
22# You should have received a copy of the GNU General Public License along
23# with this program; if not, write to the Free Software Foundation, Inc.,
24# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25#
26# Based on functions from the base bb module, Copyright 2003 Holger Schurig
27
28import os
29import bb
30from bb import data
31from bb.fetch import Fetch
32from bb.fetch import FetchError
33from bb.fetch import uri_replace
34
35class Wget(Fetch):
36 """Class to fetch urls via 'wget'"""
37 def supports(self, url, ud, d):
38 """
39 Check to see if a given url can be fetched with wget.
40 """
41 return ud.type in ['http','https','ftp']
42
43 def localpath(self, url, ud, d):
44
45 url = bb.encodeurl([ud.type, ud.host, ud.path, ud.user, ud.pswd, {}])
46 ud.basename = os.path.basename(ud.path)
47 ud.localfile = data.expand(os.path.basename(url), d)
48
49 return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
50
51 def go(self, uri, ud, d, checkonly = False):
52 """Fetch urls"""
53
54 def fetch_uri(uri, ud, d):
55 if checkonly:
56 fetchcmd = data.getVar("CHECKCOMMAND", d, 1)
57 elif os.path.exists(ud.localpath):
58 # file exists, but we didnt complete it.. trying again..
59 fetchcmd = data.getVar("RESUMECOMMAND", d, 1)
60 else:
61 fetchcmd = data.getVar("FETCHCOMMAND", d, 1)
62
63 uri = uri.split(";")[0]
64 uri_decoded = list(bb.decodeurl(uri))
65 uri_type = uri_decoded[0]
66 uri_host = uri_decoded[1]
67
68 bb.msg.note(1, bb.msg.domain.Fetcher, "fetch " + uri)
69 fetchcmd = fetchcmd.replace("${URI}", uri.split(";")[0])
70 fetchcmd = fetchcmd.replace("${FILE}", ud.basename)
71 httpproxy = None
72 ftpproxy = None
73 if uri_type == 'http':
74 httpproxy = data.getVar("HTTP_PROXY", d, True)
75 httpproxy_ignore = (data.getVar("HTTP_PROXY_IGNORE", d, True) or "").split()
76 for p in httpproxy_ignore:
77 if uri_host.endswith(p):
78 httpproxy = None
79 break
80 if uri_type == 'ftp':
81 ftpproxy = data.getVar("FTP_PROXY", d, True)
82 ftpproxy_ignore = (data.getVar("HTTP_PROXY_IGNORE", d, True) or "").split()
83 for p in ftpproxy_ignore:
84 if uri_host.endswith(p):
85 ftpproxy = None
86 break
87 if httpproxy:
88 fetchcmd = "http_proxy=" + httpproxy + " " + fetchcmd
89 if ftpproxy:
90 fetchcmd = "ftp_proxy=" + ftpproxy + " " + fetchcmd
91 bb.msg.debug(2, bb.msg.domain.Fetcher, "executing " + fetchcmd)
92 ret = os.system(fetchcmd)
93 if ret != 0:
94 return False
95
96 # Sanity check since wget can pretend it succeed when it didn't
97 # Also, this used to happen if sourceforge sent us to the mirror page
98 if not os.path.exists(ud.localpath) and not checkonly:
99 bb.msg.debug(2, bb.msg.domain.Fetcher, "The fetch command for %s returned success but %s doesn't exist?..." % (uri, ud.localpath))
100 return False
101
102 return True
103
104 localdata = data.createCopy(d)
105 data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata)
106 data.update_data(localdata)
107
108 premirrors = [ i.split() for i in (data.getVar('PREMIRRORS', localdata, 1) or "").split('\n') if i ]
109 for (find, replace) in premirrors:
110 newuri = uri_replace(uri, find, replace, d)
111 if newuri != uri:
112 if fetch_uri(newuri, ud, localdata):
113 return True
114
115 if fetch_uri(uri, ud, localdata):
116 return True
117
118 # try mirrors
119 mirrors = [ i.split() for i in (data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ]
120 for (find, replace) in mirrors:
121 newuri = uri_replace(uri, find, replace, d)
122 if newuri != uri:
123 if fetch_uri(newuri, ud, localdata):
124 return True
125
126 raise FetchError(uri)
127
128
129 def checkstatus(self, uri, ud, d):
130 return self.go(uri, ud, d, True)
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2026-02-13 09:36:15 +0000