diff options
Diffstat (limited to 'bitbake/lib/bb/fetch/__init__.py')
-rw-r--r-- | bitbake/lib/bb/fetch/__init__.py | 103 |
1 files changed, 43 insertions, 60 deletions
diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py index f235526452..f739245bd1 100644 --- a/bitbake/lib/bb/fetch/__init__.py +++ b/bitbake/lib/bb/fetch/__init__.py | |||
@@ -80,6 +80,7 @@ def uri_replace(uri, uri_find, uri_replace, d): | |||
80 | return bb.encodeurl(result_decoded) | 80 | return bb.encodeurl(result_decoded) |
81 | 81 | ||
82 | methods = [] | 82 | methods = [] |
83 | urldata_cache = {} | ||
83 | 84 | ||
84 | def fetcher_init(d): | 85 | def fetcher_init(d): |
85 | """ | 86 | """ |
@@ -87,12 +88,16 @@ def fetcher_init(d): | |||
87 | Calls before this must not hit the cache. | 88 | Calls before this must not hit the cache. |
88 | """ | 89 | """ |
89 | pd = persist_data.PersistData(d) | 90 | pd = persist_data.PersistData(d) |
90 | # Clear any cached url data | 91 | # When to drop SCM head revisions controled by user policy |
91 | pd.delDomain("BB_URLDATA") | 92 | srcrev_policy = bb.data.getVar('BB_SRCREV_POLICY', d, 1) or "clear" |
92 | # When to drop SCM head revisions should be controled by user policy | 93 | if srcrev_policy == "cache": |
93 | pd.delDomain("BB_URI_HEADREVS") | 94 | bb.msg.debug(1, bb.msg.domain.Fetcher, "Keeping SRCREV cache due to cache policy of: %s" % srcrev_policy) |
95 | elif srcrev_policy == "clear": | ||
96 | bb.msg.debug(1, bb.msg.domain.Fetcher, "Clearing SRCREV cache due to cache policy of: %s" % srcrev_policy) | ||
97 | pd.delDomain("BB_URI_HEADREVS") | ||
98 | else: | ||
99 | bb.msg.fatal(bb.msg.domain.Fetcher, "Invalid SRCREV cache policy of: %s" % srcrev_policy) | ||
94 | # Make sure our domains exist | 100 | # Make sure our domains exist |
95 | pd.addDomain("BB_URLDATA") | ||
96 | pd.addDomain("BB_URI_HEADREVS") | 101 | pd.addDomain("BB_URI_HEADREVS") |
97 | pd.addDomain("BB_URI_LOCALCOUNT") | 102 | pd.addDomain("BB_URI_LOCALCOUNT") |
98 | 103 | ||
@@ -102,45 +107,30 @@ def fetcher_init(d): | |||
102 | # 3. localpaths | 107 | # 3. localpaths |
103 | # localpath can be called at any time | 108 | # localpath can be called at any time |
104 | 109 | ||
105 | def init(urls, d, cache = True): | 110 | def init(urls, d, setup = True): |
106 | urldata = {} | 111 | urldata = {} |
107 | 112 | fn = bb.data.getVar('FILE', d, 1) | |
108 | if cache: | 113 | if fn in urldata_cache: |
109 | urldata = getdata(d) | 114 | urldata = urldata_cache[fn] |
110 | 115 | ||
111 | for url in urls: | 116 | for url in urls: |
112 | if url not in urldata: | 117 | if url not in urldata: |
113 | ud = FetchData(url, d) | 118 | urldata[url] = FetchData(url, d) |
114 | for m in methods: | ||
115 | if m.supports(url, ud, d): | ||
116 | ud.init(m, d) | ||
117 | ud.setup_localpath(d) | ||
118 | break | ||
119 | urldata[url] = ud | ||
120 | |||
121 | if cache: | ||
122 | fn = bb.data.getVar('FILE', d, 1) | ||
123 | pd = persist_data.PersistData(d) | ||
124 | pd.setValue("BB_URLDATA", fn, pickle.dumps(urldata, 0)) | ||
125 | 119 | ||
126 | return urldata | 120 | if setup: |
127 | 121 | for url in urldata: | |
128 | def getdata(d): | 122 | if not urldata[url].setup: |
129 | urldata = {} | 123 | urldata[url].setup_localpath(d) |
130 | fn = bb.data.getVar('FILE', d, 1) | ||
131 | pd = persist_data.PersistData(d) | ||
132 | encdata = pd.getValue("BB_URLDATA", fn) | ||
133 | if encdata: | ||
134 | urldata = pickle.loads(str(encdata)) | ||
135 | 124 | ||
125 | urldata_cache[fn] = urldata | ||
136 | return urldata | 126 | return urldata |
137 | 127 | ||
138 | def go(d, urldata = None): | 128 | def go(d): |
139 | """ | 129 | """ |
140 | Fetch all urls | 130 | Fetch all urls |
131 | init must have previously been called | ||
141 | """ | 132 | """ |
142 | if not urldata: | 133 | urldata = init([], d, True) |
143 | urldata = getdata(d) | ||
144 | 134 | ||
145 | for u in urldata: | 135 | for u in urldata: |
146 | ud = urldata[u] | 136 | ud = urldata[u] |
@@ -154,13 +144,12 @@ def go(d, urldata = None): | |||
154 | if ud.localfile and not m.forcefetch(u, ud, d): | 144 | if ud.localfile and not m.forcefetch(u, ud, d): |
155 | Fetch.write_md5sum(u, ud, d) | 145 | Fetch.write_md5sum(u, ud, d) |
156 | 146 | ||
157 | def localpaths(d, urldata = None): | 147 | def localpaths(d): |
158 | """ | 148 | """ |
159 | Return a list of the local filenames, assuming successful fetch | 149 | Return a list of the local filenames, assuming successful fetch |
160 | """ | 150 | """ |
161 | local = [] | 151 | local = [] |
162 | if not urldata: | 152 | urldata = init([], d, True) |
163 | urldata = getdata(d) | ||
164 | 153 | ||
165 | for u in urldata: | 154 | for u in urldata: |
166 | ud = urldata[u] | 155 | ud = urldata[u] |
@@ -177,25 +166,14 @@ def get_srcrev(d): | |||
177 | have been set. | 166 | have been set. |
178 | """ | 167 | """ |
179 | scms = [] | 168 | scms = [] |
180 | urldata = getdata(d) | 169 | # Only call setup_localpath on URIs which suppports_srcrev() |
181 | if len(urldata) == 0: | 170 | urldata = init(bb.data.getVar('SRC_URI', d, 1).split(), d, False) |
182 | src_uri = bb.data.getVar('SRC_URI', d, 1).split() | 171 | for u in urldata: |
183 | for url in src_uri: | 172 | ud = urldata[u] |
184 | if url not in urldata: | 173 | if ud.method.suppports_srcrev(): |
185 | ud = FetchData(url, d) | 174 | if not ud.setup: |
186 | for m in methods: | 175 | ud.setup_localpath(d) |
187 | if m.supports(url, ud, d): | 176 | scms.append(u) |
188 | ud.init(m, d) | ||
189 | break | ||
190 | urldata[url] = ud | ||
191 | if ud.method.suppports_srcrev(): | ||
192 | scms.append(url) | ||
193 | ud.setup_localpath(d) | ||
194 | else: | ||
195 | for u in urldata: | ||
196 | ud = urldata[u] | ||
197 | if ud.method.suppports_srcrev(): | ||
198 | scms.append(u) | ||
199 | 177 | ||
200 | if len(scms) == 0: | 178 | if len(scms) == 0: |
201 | bb.msg.error(bb.msg.domain.Fetcher, "SRCREV was used yet no valid SCM was found in SRC_URI") | 179 | bb.msg.error(bb.msg.domain.Fetcher, "SRCREV was used yet no valid SCM was found in SRC_URI") |
@@ -212,7 +190,7 @@ def localpath(url, d, cache = True): | |||
212 | Called from the parser with cache=False since the cache isn't ready | 190 | Called from the parser with cache=False since the cache isn't ready |
213 | at this point. Also called from classed in OE e.g. patch.bbclass | 191 | at this point. Also called from classed in OE e.g. patch.bbclass |
214 | """ | 192 | """ |
215 | ud = init([url], d, cache) | 193 | ud = init([url], d) |
216 | if ud[url].method: | 194 | if ud[url].method: |
217 | return ud[url].localpath | 195 | return ud[url].localpath |
218 | return url | 196 | return url |
@@ -252,17 +230,22 @@ def runfetchcmd(cmd, d, quiet = False): | |||
252 | return output | 230 | return output |
253 | 231 | ||
254 | class FetchData(object): | 232 | class FetchData(object): |
255 | """Class for fetcher variable store""" | 233 | """ |
234 | A class which represents the fetcher state for a given URI. | ||
235 | """ | ||
256 | def __init__(self, url, d): | 236 | def __init__(self, url, d): |
257 | self.localfile = "" | 237 | self.localfile = "" |
258 | (self.type, self.host, self.path, self.user, self.pswd, self.parm) = bb.decodeurl(data.expand(url, d)) | 238 | (self.type, self.host, self.path, self.user, self.pswd, self.parm) = bb.decodeurl(data.expand(url, d)) |
259 | self.date = Fetch.getSRCDate(self, d) | 239 | self.date = Fetch.getSRCDate(self, d) |
260 | self.url = url | 240 | self.url = url |
261 | 241 | self.setup = False | |
262 | def init(self, method, d): | 242 | for m in methods: |
263 | self.method = method | 243 | if m.supports(url, self, d): |
244 | self.method = m | ||
245 | break | ||
264 | 246 | ||
265 | def setup_localpath(self, d): | 247 | def setup_localpath(self, d): |
248 | self.setup = True | ||
266 | if "localpath" in self.parm: | 249 | if "localpath" in self.parm: |
267 | self.localpath = self.parm["localpath"] | 250 | self.localpath = self.parm["localpath"] |
268 | else: | 251 | else: |