diff options
Diffstat (limited to 'bitbake/lib/bb/fetch/__init__.py')
-rw-r--r-- | bitbake/lib/bb/fetch/__init__.py | 167 |
1 files changed, 112 insertions, 55 deletions
diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py index 7ab0590765..24aebc41ca 100644 --- a/bitbake/lib/bb/fetch/__init__.py +++ b/bitbake/lib/bb/fetch/__init__.py | |||
@@ -38,13 +38,16 @@ class NoMethodError(Exception): | |||
38 | class MissingParameterError(Exception): | 38 | class MissingParameterError(Exception): |
39 | """Exception raised when a fetch method is missing a critical parameter in the url""" | 39 | """Exception raised when a fetch method is missing a critical parameter in the url""" |
40 | 40 | ||
41 | class ParameterError(Exception): | ||
42 | """Exception raised when a url cannot be proccessed due to invalid parameters.""" | ||
43 | |||
41 | class MD5SumError(Exception): | 44 | class MD5SumError(Exception): |
42 | """Exception raised when a MD5SUM of a file does not match the expected one""" | 45 | """Exception raised when a MD5SUM of a file does not match the expected one""" |
43 | 46 | ||
44 | def uri_replace(uri, uri_find, uri_replace, d): | 47 | def uri_replace(uri, uri_find, uri_replace, d): |
45 | # bb.note("uri_replace: operating on %s" % uri) | 48 | # bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: operating on %s" % uri) |
46 | if not uri or not uri_find or not uri_replace: | 49 | if not uri or not uri_find or not uri_replace: |
47 | bb.debug(1, "uri_replace: passed an undefined value, not replacing") | 50 | bb.msg.debug(1, bb.msg.domain.Fetcher, "uri_replace: passed an undefined value, not replacing") |
48 | uri_decoded = list(bb.decodeurl(uri)) | 51 | uri_decoded = list(bb.decodeurl(uri)) |
49 | uri_find_decoded = list(bb.decodeurl(uri_find)) | 52 | uri_find_decoded = list(bb.decodeurl(uri_find)) |
50 | uri_replace_decoded = list(bb.decodeurl(uri_replace)) | 53 | uri_replace_decoded = list(bb.decodeurl(uri_replace)) |
@@ -62,9 +65,9 @@ def uri_replace(uri, uri_find, uri_replace, d): | |||
62 | localfn = bb.fetch.localpath(uri, d) | 65 | localfn = bb.fetch.localpath(uri, d) |
63 | if localfn: | 66 | if localfn: |
64 | result_decoded[loc] = os.path.dirname(result_decoded[loc]) + "/" + os.path.basename(bb.fetch.localpath(uri, d)) | 67 | result_decoded[loc] = os.path.dirname(result_decoded[loc]) + "/" + os.path.basename(bb.fetch.localpath(uri, d)) |
65 | # bb.note("uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc])) | 68 | # bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc])) |
66 | else: | 69 | else: |
67 | # bb.note("uri_replace: no match") | 70 | # bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: no match") |
68 | return uri | 71 | return uri |
69 | # else: | 72 | # else: |
70 | # for j in i.keys(): | 73 | # for j in i.keys(): |
@@ -72,62 +75,94 @@ def uri_replace(uri, uri_find, uri_replace, d): | |||
72 | return bb.encodeurl(result_decoded) | 75 | return bb.encodeurl(result_decoded) |
73 | 76 | ||
74 | methods = [] | 77 | methods = [] |
78 | urldata = {} | ||
75 | 79 | ||
76 | def init(urls = [], d = None): | 80 | def init(urls = [], d = None): |
77 | if d == None: | 81 | if d == None: |
78 | bb.debug(2,"BUG init called with None as data object!!!") | 82 | bb.msg.debug(2, bb.msg.domain.Fetcher, "BUG init called with None as data object!!!") |
79 | return | 83 | return |
80 | 84 | ||
81 | for m in methods: | 85 | for m in methods: |
82 | m.urls = [] | 86 | m.urls = [] |
83 | 87 | ||
84 | for u in urls: | 88 | for u in urls: |
89 | ud = initdata(u, d) | ||
90 | if ud.method: | ||
91 | ud.method.urls.append(u) | ||
92 | |||
93 | def initdata(url, d): | ||
94 | if url not in urldata: | ||
95 | ud = FetchData() | ||
96 | (ud.type, ud.host, ud.path, ud.user, ud.pswd, ud.parm) = bb.decodeurl(data.expand(url, d)) | ||
97 | ud.date = Fetch.getSRCDate(d) | ||
85 | for m in methods: | 98 | for m in methods: |
86 | m.data = d | 99 | if m.supports(url, ud, d): |
87 | if m.supports(u, d): | 100 | ud.localpath = m.localpath(url, ud, d) |
88 | m.urls.append(u) | 101 | ud.md5 = ud.localpath + '.md5' |
102 | # if user sets localpath for file, use it instead. | ||
103 | if "localpath" in ud.parm: | ||
104 | ud.localpath = ud.parm["localpath"] | ||
105 | ud.method = m | ||
106 | break | ||
107 | urldata[url] = ud | ||
108 | return urldata[url] | ||
89 | 109 | ||
90 | def go(d): | 110 | def go(d): |
91 | """Fetch all urls""" | 111 | """Fetch all urls""" |
92 | for m in methods: | 112 | for m in methods: |
93 | if m.urls: | 113 | for u in m.urls: |
94 | m.go(d) | 114 | ud = urldata[u] |
115 | if ud.localfile and not m.forcefetch(u, ud, d) and os.path.exists(urldata[u].md5): | ||
116 | # File already present along with md5 stamp file | ||
117 | # Touch md5 file to show activity | ||
118 | os.utime(ud.md5, None) | ||
119 | continue | ||
120 | # RP - is olddir needed? | ||
121 | # olddir = os.path.abspath(os.getcwd()) | ||
122 | m.go(u, ud , d) | ||
123 | # os.chdir(olddir) | ||
124 | if ud.localfile and not m.forcefetch(u, ud, d): | ||
125 | Fetch.write_md5sum(u, ud, d) | ||
95 | 126 | ||
96 | def localpaths(d): | 127 | def localpaths(d): |
97 | """Return a list of the local filenames, assuming successful fetch""" | 128 | """Return a list of the local filenames, assuming successful fetch""" |
98 | local = [] | 129 | local = [] |
99 | for m in methods: | 130 | for m in methods: |
100 | for u in m.urls: | 131 | for u in m.urls: |
101 | local.append(m.localpath(u, d)) | 132 | local.append(urldata[u].localpath) |
102 | return local | 133 | return local |
103 | 134 | ||
104 | def localpath(url, d): | 135 | def localpath(url, d): |
105 | for m in methods: | 136 | ud = initdata(url, d) |
106 | if m.supports(url, d): | 137 | if ud.method: |
107 | return m.localpath(url, d) | 138 | return ud.localpath |
108 | return url | 139 | return url |
109 | 140 | ||
141 | class FetchData(object): | ||
142 | """Class for fetcher variable store""" | ||
143 | def __init__(self): | ||
144 | self.localfile = "" | ||
145 | |||
146 | |||
110 | class Fetch(object): | 147 | class Fetch(object): |
111 | """Base class for 'fetch'ing data""" | 148 | """Base class for 'fetch'ing data""" |
112 | 149 | ||
113 | def __init__(self, urls = []): | 150 | def __init__(self, urls = []): |
114 | self.urls = [] | 151 | self.urls = [] |
115 | for url in urls: | ||
116 | if self.supports(bb.decodeurl(url), d) is 1: | ||
117 | self.urls.append(url) | ||
118 | 152 | ||
119 | def supports(url, d): | 153 | def supports(self, url, urldata, d): |
120 | """Check to see if this fetch class supports a given url. | 154 | """ |
121 | Expects supplied url in list form, as outputted by bb.decodeurl(). | 155 | Check to see if this fetch class supports a given url. |
122 | """ | 156 | """ |
123 | return 0 | 157 | return 0 |
124 | supports = staticmethod(supports) | ||
125 | 158 | ||
126 | def localpath(url, d): | 159 | def localpath(self, url, urldata, d): |
127 | """Return the local filename of a given url assuming a successful fetch. | 160 | """ |
161 | Return the local filename of a given url assuming a successful fetch. | ||
162 | Can also setup variables in urldata for use in go (saving code duplication | ||
163 | and duplicate code execution) | ||
128 | """ | 164 | """ |
129 | return url | 165 | return url |
130 | localpath = staticmethod(localpath) | ||
131 | 166 | ||
132 | def setUrls(self, urls): | 167 | def setUrls(self, urls): |
133 | self.__urls = urls | 168 | self.__urls = urls |
@@ -137,16 +172,17 @@ class Fetch(object): | |||
137 | 172 | ||
138 | urls = property(getUrls, setUrls, None, "Urls property") | 173 | urls = property(getUrls, setUrls, None, "Urls property") |
139 | 174 | ||
140 | def setData(self, data): | 175 | def forcefetch(self, url, urldata, d): |
141 | self.__data = data | 176 | """ |
142 | 177 | Force a fetch, even if localpath exists? | |
143 | def getData(self): | 178 | """ |
144 | return self.__data | 179 | return False |
145 | |||
146 | data = property(getData, setData, None, "Data property") | ||
147 | 180 | ||
148 | def go(self, urls = []): | 181 | def go(self, url, urldata, d): |
149 | """Fetch urls""" | 182 | """ |
183 | Fetch urls | ||
184 | Assumes localpath was called first | ||
185 | """ | ||
150 | raise NoMethodError("Missing implementation for url") | 186 | raise NoMethodError("Missing implementation for url") |
151 | 187 | ||
152 | def getSRCDate(d): | 188 | def getSRCDate(d): |
@@ -155,7 +191,12 @@ class Fetch(object): | |||
155 | 191 | ||
156 | d the bb.data module | 192 | d the bb.data module |
157 | """ | 193 | """ |
158 | return data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1 ) | 194 | pn = data.getVar("PN", d, 1) |
195 | |||
196 | if pn: | ||
197 | return data.getVar("SRCDATE_%s" % pn, d, 1) or data.getVar("CVSDATE_%s" % pn, d, 1) or data.getVar("DATE", d, 1) | ||
198 | |||
199 | return data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1) | ||
159 | getSRCDate = staticmethod(getSRCDate) | 200 | getSRCDate = staticmethod(getSRCDate) |
160 | 201 | ||
161 | def try_mirror(d, tarfn): | 202 | def try_mirror(d, tarfn): |
@@ -168,6 +209,11 @@ class Fetch(object): | |||
168 | d Is a bb.data instance | 209 | d Is a bb.data instance |
169 | tarfn is the name of the tarball | 210 | tarfn is the name of the tarball |
170 | """ | 211 | """ |
212 | tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn) | ||
213 | if os.access(tarpath, os.R_OK): | ||
214 | bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % tarfn) | ||
215 | return True | ||
216 | |||
171 | pn = data.getVar('PN', d, True) | 217 | pn = data.getVar('PN', d, True) |
172 | src_tarball_stash = None | 218 | src_tarball_stash = None |
173 | if pn: | 219 | if pn: |
@@ -176,36 +222,45 @@ class Fetch(object): | |||
176 | for stash in src_tarball_stash: | 222 | for stash in src_tarball_stash: |
177 | fetchcmd = data.getVar("FETCHCOMMAND_mirror", d, True) or data.getVar("FETCHCOMMAND_wget", d, True) | 223 | fetchcmd = data.getVar("FETCHCOMMAND_mirror", d, True) or data.getVar("FETCHCOMMAND_wget", d, True) |
178 | uri = stash + tarfn | 224 | uri = stash + tarfn |
179 | bb.note("fetch " + uri) | 225 | bb.msg.note(1, bb.msg.domain.Fetcher, "fetch " + uri) |
180 | fetchcmd = fetchcmd.replace("${URI}", uri) | 226 | fetchcmd = fetchcmd.replace("${URI}", uri) |
181 | ret = os.system(fetchcmd) | 227 | ret = os.system(fetchcmd) |
182 | if ret == 0: | 228 | if ret == 0: |
183 | bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn) | 229 | bb.msg.note(1, bb.msg.domain.Fetcher, "Fetched %s from tarball stash, skipping checkout" % tarfn) |
184 | return True | 230 | return True |
185 | return False | 231 | return False |
186 | try_mirror = staticmethod(try_mirror) | 232 | try_mirror = staticmethod(try_mirror) |
187 | 233 | ||
188 | def check_for_tarball(d, tarfn, dldir, date): | 234 | def verify_md5sum(ud, got_sum): |
189 | """ | 235 | """ |
190 | Check for a local copy then check the tarball stash. | 236 | Verify the md5sum we wanted with the one we got |
191 | Both checks are skipped if date == 'now'. | ||
192 | |||
193 | d Is a bb.data instance | ||
194 | tarfn is the name of the tarball | ||
195 | date is the SRCDATE | ||
196 | """ | 237 | """ |
197 | if "now" != date: | 238 | wanted_sum = None |
198 | dl = os.path.join(dldir, tarfn) | 239 | if 'md5sum' in ud.parm: |
199 | if os.access(dl, os.R_OK): | 240 | wanted_sum = ud.parm['md5sum'] |
200 | bb.debug(1, "%s already exists, skipping checkout." % tarfn) | 241 | if not wanted_sum: |
201 | return True | 242 | return True |
202 | 243 | ||
203 | # try to use the tarball stash | 244 | return wanted_sum == got_sum |
204 | if Fetch.try_mirror(d, tarfn): | 245 | verify_md5sum = staticmethod(verify_md5sum) |
205 | return True | 246 | |
206 | return False | 247 | def write_md5sum(url, ud, d): |
207 | check_for_tarball = staticmethod(check_for_tarball) | 248 | if bb.which(data.getVar('PATH', d), 'md5sum'): |
208 | 249 | try: | |
250 | md5pipe = os.popen('md5sum ' + ud.localpath) | ||
251 | md5data = (md5pipe.readline().split() or [ "" ])[0] | ||
252 | md5pipe.close() | ||
253 | except OSError: | ||
254 | md5data = "" | ||
255 | |||
256 | # verify the md5sum | ||
257 | if not Fetch.verify_md5sum(ud, md5data): | ||
258 | raise MD5SumError(url) | ||
259 | |||
260 | md5out = file(ud.md5, 'w') | ||
261 | md5out.write(md5data) | ||
262 | md5out.close() | ||
263 | write_md5sum = staticmethod(write_md5sum) | ||
209 | 264 | ||
210 | import cvs | 265 | import cvs |
211 | import git | 266 | import git |
@@ -214,6 +269,7 @@ import svn | |||
214 | import wget | 269 | import wget |
215 | import svk | 270 | import svk |
216 | import ssh | 271 | import ssh |
272 | import perforce | ||
217 | 273 | ||
218 | methods.append(cvs.Cvs()) | 274 | methods.append(cvs.Cvs()) |
219 | methods.append(git.Git()) | 275 | methods.append(git.Git()) |
@@ -222,3 +278,4 @@ methods.append(svn.Svn()) | |||
222 | methods.append(wget.Wget()) | 278 | methods.append(wget.Wget()) |
223 | methods.append(svk.Svk()) | 279 | methods.append(svk.Svk()) |
224 | methods.append(ssh.SSH()) | 280 | methods.append(ssh.SSH()) |
281 | methods.append(perforce.Perforce()) | ||