diff options
Diffstat (limited to 'bitbake/lib/bb/fetch/__init__.py')
-rw-r--r-- | bitbake/lib/bb/fetch/__init__.py | 301 |
1 files changed, 236 insertions, 65 deletions
diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py index 31a4adccb1..6ebf5a34ad 100644 --- a/bitbake/lib/bb/fetch/__init__.py +++ b/bitbake/lib/bb/fetch/__init__.py | |||
@@ -27,6 +27,12 @@ BitBake build tools. | |||
27 | import os, re | 27 | import os, re |
28 | import bb | 28 | import bb |
29 | from bb import data | 29 | from bb import data |
30 | from bb import persist_data | ||
31 | |||
32 | try: | ||
33 | import cPickle as pickle | ||
34 | except ImportError: | ||
35 | import pickle | ||
30 | 36 | ||
31 | class FetchError(Exception): | 37 | class FetchError(Exception): |
32 | """Exception raised when a download fails""" | 38 | """Exception raised when a download fails""" |
@@ -74,78 +80,193 @@ def uri_replace(uri, uri_find, uri_replace, d): | |||
74 | return bb.encodeurl(result_decoded) | 80 | return bb.encodeurl(result_decoded) |
75 | 81 | ||
76 | methods = [] | 82 | methods = [] |
77 | urldata = {} | ||
78 | |||
79 | def init(urls = [], d = None): | ||
80 | if d == None: | ||
81 | bb.msg.debug(2, bb.msg.domain.Fetcher, "BUG init called with None as data object!!!") | ||
82 | return | ||
83 | |||
84 | for m in methods: | ||
85 | m.urls = [] | ||
86 | 83 | ||
87 | for u in urls: | 84 | def fetcher_init(d): |
88 | ud = initdata(u, d) | 85 | """ |
89 | if ud.method: | 86 | Called to initilize the fetchers once the configuration data is known |
90 | ud.method.urls.append(u) | 87 | Calls before this must not hit the cache. |
91 | 88 | """ | |
92 | def initdata(url, d): | 89 | pd = persist_data.PersistData(d) |
93 | fn = bb.data.getVar('FILE', d, 1) | 90 | # Clear any cached url data |
94 | if fn not in urldata: | 91 | pd.delDomain("BB_URLDATA") |
95 | urldata[fn] = {} | 92 | # When to drop SCM head revisions should be controled by user policy |
96 | if url not in urldata[fn]: | 93 | pd.delDomain("BB_URI_HEADREVS") |
97 | ud = FetchData() | 94 | # Make sure our domains exist |
98 | (ud.type, ud.host, ud.path, ud.user, ud.pswd, ud.parm) = bb.decodeurl(data.expand(url, d)) | 95 | pd.addDomain("BB_URLDATA") |
99 | ud.date = Fetch.getSRCDate(ud, d) | 96 | pd.addDomain("BB_URI_HEADREVS") |
100 | for m in methods: | 97 | pd.addDomain("BB_URI_LOCALCOUNT") |
101 | if m.supports(url, ud, d): | 98 | |
102 | ud.localpath = m.localpath(url, ud, d) | 99 | # Function call order is usually: |
103 | ud.md5 = ud.localpath + '.md5' | 100 | # 1. init |
104 | # if user sets localpath for file, use it instead. | 101 | # 2. go |
105 | if "localpath" in ud.parm: | 102 | # 3. localpaths |
106 | ud.localpath = ud.parm["localpath"] | 103 | # localpath can be called at any time |
107 | ud.method = m | 104 | |
108 | break | 105 | def init(urls, d, cache = True): |
109 | urldata[fn][url] = ud | 106 | urldata = {} |
110 | return urldata[fn][url] | 107 | |
111 | 108 | if cache: | |
112 | def go(d): | 109 | urldata, pd, fn = getdata(d) |
113 | """Fetch all urls""" | 110 | |
111 | for url in urls: | ||
112 | if url not in urldata: | ||
113 | ud = FetchData(url, d) | ||
114 | for m in methods: | ||
115 | if m.supports(url, ud, d): | ||
116 | ud.init(m, d) | ||
117 | ud.setup_localpath(d) | ||
118 | break | ||
119 | urldata[url] = ud | ||
120 | |||
121 | if cache: | ||
122 | pd.setValue("BB_URLDATA", fn, pickle.dumps(urldata, 0)) | ||
123 | |||
124 | return urldata | ||
125 | |||
126 | def getdata(d): | ||
127 | urldata = {} | ||
114 | fn = bb.data.getVar('FILE', d, 1) | 128 | fn = bb.data.getVar('FILE', d, 1) |
115 | for m in methods: | 129 | pd = persist_data.PersistData(d) |
116 | for u in m.urls: | 130 | encdata = pd.getValue("BB_URLDATA", fn) |
117 | ud = urldata[fn][u] | 131 | if encdata: |
118 | if ud.localfile and not m.forcefetch(u, ud, d) and os.path.exists(urldata[fn][u].md5): | 132 | urldata = pickle.loads(str(encdata)) |
119 | # File already present along with md5 stamp file | 133 | |
120 | # Touch md5 file to show activity | 134 | return urldata, pd, fn |
121 | os.utime(ud.md5, None) | 135 | |
122 | continue | 136 | def go(d, urldata = None): |
123 | # RP - is olddir needed? | 137 | """ |
124 | # olddir = os.path.abspath(os.getcwd()) | 138 | Fetch all urls |
125 | m.go(u, ud , d) | 139 | """ |
126 | # os.chdir(olddir) | 140 | if not urldata: |
127 | if ud.localfile and not m.forcefetch(u, ud, d): | 141 | urldata, pd, fn = getdata(d) |
128 | Fetch.write_md5sum(u, ud, d) | 142 | |
129 | 143 | for u in urldata: | |
130 | def localpaths(d): | 144 | ud = urldata[u] |
131 | """Return a list of the local filenames, assuming successful fetch""" | 145 | m = ud.method |
146 | if ud.localfile and not m.forcefetch(u, ud, d) and os.path.exists(ud.md5): | ||
147 | # File already present along with md5 stamp file | ||
148 | # Touch md5 file to show activity | ||
149 | os.utime(ud.md5, None) | ||
150 | continue | ||
151 | m.go(u, ud, d) | ||
152 | if ud.localfile and not m.forcefetch(u, ud, d): | ||
153 | Fetch.write_md5sum(u, ud, d) | ||
154 | |||
155 | def localpaths(d, urldata = None): | ||
156 | """ | ||
157 | Return a list of the local filenames, assuming successful fetch | ||
158 | """ | ||
132 | local = [] | 159 | local = [] |
133 | fn = bb.data.getVar('FILE', d, 1) | 160 | if not urldata: |
134 | for m in methods: | 161 | urldata, pd, fn = getdata(d) |
135 | for u in m.urls: | 162 | |
136 | local.append(urldata[fn][u].localpath) | 163 | for u in urldata: |
164 | ud = urldata[u] | ||
165 | local.append(ud.localpath) | ||
166 | |||
137 | return local | 167 | return local |
138 | 168 | ||
139 | def localpath(url, d): | 169 | def get_srcrev(d): |
140 | ud = initdata(url, d) | 170 | """ |
141 | if ud.method: | 171 | Return the version string for the current package |
142 | return ud.localpath | 172 | (usually to be used as PV) |
173 | Most packages usually only have one SCM so we just pass on the call. | ||
174 | In the multi SCM case, we build a value based on SRCREV_FORMAT which must | ||
175 | have been set. | ||
176 | """ | ||
177 | scms = [] | ||
178 | urldata, pd, fn = getdata(d) | ||
179 | if len(urldata) == 0: | ||
180 | src_uri = bb.data.getVar('SRC_URI', d, 1).split() | ||
181 | for url in src_uri: | ||
182 | if url not in urldata: | ||
183 | ud = FetchData(url, d) | ||
184 | for m in methods: | ||
185 | if m.supports(url, ud, d): | ||
186 | ud.init(m, d) | ||
187 | break | ||
188 | urldata[url] = ud | ||
189 | if ud.method.suppports_srcrev(): | ||
190 | scms.append(url) | ||
191 | ud.setup_localpath(d) | ||
192 | else: | ||
193 | for u in urldata: | ||
194 | ud = urldata[u] | ||
195 | if ud.method.suppports_srcrev(): | ||
196 | scms.append(u) | ||
197 | |||
198 | if len(scms) == 0: | ||
199 | bb.msg.error(bb.msg.domain.Fetcher, "SRCREV was used yet no valid SCM was found in SRC_URI") | ||
200 | raise ParameterError | ||
201 | |||
202 | if len(scms) == 1: | ||
203 | return urldata[scms[0]].method.sortable_revision(scms[0], urldata[scms[0]], d) | ||
204 | |||
205 | bb.msg.error(bb.msg.domain.Fetcher, "Sorry, support for SRCREV_FORMAT still needs to be written") | ||
206 | raise ParameterError | ||
207 | |||
208 | def localpath(url, d, cache = True): | ||
209 | """ | ||
210 | Called from the parser with cache=False since the cache isn't ready | ||
211 | at this point. Also called from classed in OE e.g. patch.bbclass | ||
212 | """ | ||
213 | ud = init([url], d, cache) | ||
214 | if ud[url].method: | ||
215 | return ud[url].localpath | ||
143 | return url | 216 | return url |
144 | 217 | ||
218 | def runfetchcmd(cmd, d, quiet = False): | ||
219 | """ | ||
220 | Run cmd returning the command output | ||
221 | Raise an error if interrupted or cmd fails | ||
222 | Optionally echo command output to stdout | ||
223 | """ | ||
224 | bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % cmd) | ||
225 | |||
226 | # Need to export PATH as binary could be in metadata paths | ||
227 | # rather than host provided | ||
228 | pathcmd = 'export PATH=%s; %s' % (data.expand('${PATH}', d), cmd) | ||
229 | |||
230 | stdout_handle = os.popen(pathcmd, "r") | ||
231 | output = "" | ||
232 | |||
233 | while 1: | ||
234 | line = stdout_handle.readline() | ||
235 | if not line: | ||
236 | break | ||
237 | if not quiet: | ||
238 | print line | ||
239 | output += line | ||
240 | |||
241 | status = stdout_handle.close() or 0 | ||
242 | signal = status >> 8 | ||
243 | exitstatus = status & 0xff | ||
244 | |||
245 | if signal: | ||
246 | raise FetchError("Fetch command %s failed with signal %s, output:\n%s" % (pathcmd, signal, output)) | ||
247 | elif status != 0: | ||
248 | raise FetchError("Fetch command %s failed with exit code %s, output:\n%s" % (pathcmd, status, output)) | ||
249 | |||
250 | return output | ||
251 | |||
145 | class FetchData(object): | 252 | class FetchData(object): |
146 | """Class for fetcher variable store""" | 253 | """Class for fetcher variable store""" |
147 | def __init__(self): | 254 | def __init__(self, url, d): |
148 | self.localfile = "" | 255 | self.localfile = "" |
256 | (self.type, self.host, self.path, self.user, self.pswd, self.parm) = bb.decodeurl(data.expand(url, d)) | ||
257 | self.date = Fetch.getSRCDate(self, d) | ||
258 | self.url = url | ||
259 | |||
260 | def init(self, method, d): | ||
261 | self.method = method | ||
262 | |||
263 | def setup_localpath(self, d): | ||
264 | if "localpath" in self.parm: | ||
265 | self.localpath = self.parm["localpath"] | ||
266 | else: | ||
267 | self.localpath = self.method.localpath(self.url, self, d) | ||
268 | self.md5 = self.localpath + '.md5' | ||
269 | # if user sets localpath for file, use it instead. | ||
149 | 270 | ||
150 | 271 | ||
151 | class Fetch(object): | 272 | class Fetch(object): |
@@ -182,6 +303,12 @@ class Fetch(object): | |||
182 | """ | 303 | """ |
183 | return False | 304 | return False |
184 | 305 | ||
306 | def suppports_srcrev(self): | ||
307 | """ | ||
308 | The fetcher supports auto source revisions (SRCREV) | ||
309 | """ | ||
310 | return False | ||
311 | |||
185 | def go(self, url, urldata, d): | 312 | def go(self, url, urldata, d): |
186 | """ | 313 | """ |
187 | Fetch urls | 314 | Fetch urls |
@@ -269,6 +396,50 @@ class Fetch(object): | |||
269 | md5out.close() | 396 | md5out.close() |
270 | write_md5sum = staticmethod(write_md5sum) | 397 | write_md5sum = staticmethod(write_md5sum) |
271 | 398 | ||
399 | def latest_revision(self, url, ud, d): | ||
400 | """ | ||
401 | Look in the cache for the latest revision, if not present ask the SCM. | ||
402 | """ | ||
403 | if not hasattr(self, "_latest_revision"): | ||
404 | raise ParameterError | ||
405 | |||
406 | pd = persist_data.PersistData(d) | ||
407 | key = self._revision_key(url, ud, d) | ||
408 | rev = pd.getValue("BB_URI_HEADREVS", key) | ||
409 | if rev != None: | ||
410 | return str(rev) | ||
411 | |||
412 | rev = self._latest_revision(url, ud, d) | ||
413 | pd.setValue("BB_URI_HEADREVS", key, rev) | ||
414 | return rev | ||
415 | |||
416 | def sortable_revision(self, url, ud, d): | ||
417 | """ | ||
418 | |||
419 | """ | ||
420 | if hasattr(self, "_sortable_revision"): | ||
421 | return self._sortable_revision(url, ud, d) | ||
422 | |||
423 | pd = persist_data.PersistData(d) | ||
424 | key = self._revision_key(url, ud, d) | ||
425 | latest_rev = self.latest_revision(url, ud, d) | ||
426 | last_rev = pd.getValue("BB_URI_LOCALCOUNT", key + "_rev") | ||
427 | count = pd.getValue("BB_URI_LOCALCOUNT", key + "_count") | ||
428 | |||
429 | if last_rev == latest_rev: | ||
430 | return str(count + "+" + latest_rev) | ||
431 | |||
432 | if count is None: | ||
433 | count = "0" | ||
434 | else: | ||
435 | count = str(int(count) + 1) | ||
436 | |||
437 | pd.setValue("BB_URI_LOCALCOUNT", key + "_rev", latest_rev) | ||
438 | pd.setValue("BB_URI_LOCALCOUNT", key + "_count", count) | ||
439 | |||
440 | return str(count + "+" + latest_rev) | ||
441 | |||
442 | |||
272 | import cvs | 443 | import cvs |
273 | import git | 444 | import git |
274 | import local | 445 | import local |
@@ -278,11 +449,11 @@ import svk | |||
278 | import ssh | 449 | import ssh |
279 | import perforce | 450 | import perforce |
280 | 451 | ||
281 | methods.append(cvs.Cvs()) | ||
282 | methods.append(git.Git()) | ||
283 | methods.append(local.Local()) | 452 | methods.append(local.Local()) |
284 | methods.append(svn.Svn()) | ||
285 | methods.append(wget.Wget()) | 453 | methods.append(wget.Wget()) |
454 | methods.append(svn.Svn()) | ||
455 | methods.append(git.Git()) | ||
456 | methods.append(cvs.Cvs()) | ||
286 | methods.append(svk.Svk()) | 457 | methods.append(svk.Svk()) |
287 | methods.append(ssh.SSH()) | 458 | methods.append(ssh.SSH()) |
288 | methods.append(perforce.Perforce()) | 459 | methods.append(perforce.Perforce()) |