summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/tests/fetch.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/tests/fetch.py')
-rw-r--r--bitbake/lib/bb/tests/fetch.py424
1 files changed, 424 insertions, 0 deletions
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
new file mode 100644
index 0000000000..4bcff543fc
--- /dev/null
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -0,0 +1,424 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3#
4# BitBake Tests for the Fetcher (fetch2/)
5#
6# Copyright (C) 2012 Richard Purdie
7#
8# This program is free software; you can redistribute it and/or modify
9# it under the terms of the GNU General Public License version 2 as
10# published by the Free Software Foundation.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License along
18# with this program; if not, write to the Free Software Foundation, Inc.,
19# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20#
21
22import unittest
23import tempfile
24import subprocess
25import os
26from bb.fetch2 import URI
27import bb
28
29class URITest(unittest.TestCase):
30 test_uris = {
31 "http://www.google.com/index.html" : {
32 'uri': 'http://www.google.com/index.html',
33 'scheme': 'http',
34 'hostname': 'www.google.com',
35 'port': None,
36 'hostport': 'www.google.com',
37 'path': '/index.html',
38 'userinfo': '',
39 'username': '',
40 'password': '',
41 'params': {},
42 'relative': False
43 },
44 "http://www.google.com/index.html;param1=value1" : {
45 'uri': 'http://www.google.com/index.html;param1=value1',
46 'scheme': 'http',
47 'hostname': 'www.google.com',
48 'port': None,
49 'hostport': 'www.google.com',
50 'path': '/index.html',
51 'userinfo': '',
52 'username': '',
53 'password': '',
54 'params': {
55 'param1': 'value1'
56 },
57 'relative': False
58 },
59 "http://www.example.com:8080/index.html" : {
60 'uri': 'http://www.example.com:8080/index.html',
61 'scheme': 'http',
62 'hostname': 'www.example.com',
63 'port': 8080,
64 'hostport': 'www.example.com:8080',
65 'path': '/index.html',
66 'userinfo': '',
67 'username': '',
68 'password': '',
69 'params': {},
70 'relative': False
71 },
72 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : {
73 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg',
74 'scheme': 'cvs',
75 'hostname': 'cvs.handhelds.org',
76 'port': None,
77 'hostport': 'cvs.handhelds.org',
78 'path': '/cvs',
79 'userinfo': 'anoncvs',
80 'username': 'anoncvs',
81 'password': '',
82 'params': {
83 'module': 'familiar/dist/ipkg'
84 },
85 'relative': False
86 },
87 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": {
88 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg',
89 'scheme': 'cvs',
90 'hostname': 'cvs.handhelds.org',
91 'port': None,
92 'hostport': 'cvs.handhelds.org',
93 'path': '/cvs',
94 'userinfo': 'anoncvs:anonymous',
95 'username': 'anoncvs',
96 'password': 'anonymous',
97 'params': {
98 'tag': 'V0-99-81',
99 'module': 'familiar/dist/ipkg'
100 },
101 'relative': False
102 },
103 "file://example.diff": { # NOTE: Not RFC compliant!
104 'uri': 'file:example.diff',
105 'scheme': 'file',
106 'hostname': '',
107 'port': None,
108 'hostport': '',
109 'path': 'example.diff',
110 'userinfo': '',
111 'username': '',
112 'password': '',
113 'params': {},
114 'relative': True
115 },
116 "file:example.diff": { # NOTE: RFC compliant version of the former
117 'uri': 'file:example.diff',
118 'scheme': 'file',
119 'hostname': '',
120 'port': None,
121 'hostport': '',
122 'path': 'example.diff',
123 'userinfo': '',
124 'userinfo': '',
125 'username': '',
126 'password': '',
127 'params': {},
128 'relative': True
129 },
130 "file:///tmp/example.diff": {
131 'uri': 'file:///tmp/example.diff',
132 'scheme': 'file',
133 'hostname': '',
134 'port': None,
135 'hostport': '',
136 'path': '/tmp/example.diff',
137 'userinfo': '',
138 'userinfo': '',
139 'username': '',
140 'password': '',
141 'params': {},
142 'relative': False
143 },
144 "git:///path/example.git": {
145 'uri': 'git:///path/example.git',
146 'scheme': 'git',
147 'hostname': '',
148 'port': None,
149 'hostport': '',
150 'path': '/path/example.git',
151 'userinfo': '',
152 'userinfo': '',
153 'username': '',
154 'password': '',
155 'params': {},
156 'relative': False
157 },
158 "git:path/example.git": {
159 'uri': 'git:path/example.git',
160 'scheme': 'git',
161 'hostname': '',
162 'port': None,
163 'hostport': '',
164 'path': 'path/example.git',
165 'userinfo': '',
166 'userinfo': '',
167 'username': '',
168 'password': '',
169 'params': {},
170 'relative': True
171 },
172 "git://example.net/path/example.git": {
173 'uri': 'git://example.net/path/example.git',
174 'scheme': 'git',
175 'hostname': 'example.net',
176 'port': None,
177 'hostport': 'example.net',
178 'path': '/path/example.git',
179 'userinfo': '',
180 'userinfo': '',
181 'username': '',
182 'password': '',
183 'params': {},
184 'relative': False
185 }
186 }
187
188 def test_uri(self):
189 for test_uri, ref in self.test_uris.items():
190 uri = URI(test_uri)
191
192 self.assertEqual(str(uri), ref['uri'])
193
194 # expected attributes
195 self.assertEqual(uri.scheme, ref['scheme'])
196
197 self.assertEqual(uri.userinfo, ref['userinfo'])
198 self.assertEqual(uri.username, ref['username'])
199 self.assertEqual(uri.password, ref['password'])
200
201 self.assertEqual(uri.hostname, ref['hostname'])
202 self.assertEqual(uri.port, ref['port'])
203 self.assertEqual(uri.hostport, ref['hostport'])
204
205 self.assertEqual(uri.path, ref['path'])
206 self.assertEqual(uri.params, ref['params'])
207
208 self.assertEqual(uri.relative, ref['relative'])
209
210 def test_dict(self):
211 for test in self.test_uris.values():
212 uri = URI()
213
214 self.assertEqual(uri.scheme, '')
215 self.assertEqual(uri.userinfo, '')
216 self.assertEqual(uri.username, '')
217 self.assertEqual(uri.password, '')
218 self.assertEqual(uri.hostname, '')
219 self.assertEqual(uri.port, None)
220 self.assertEqual(uri.path, '')
221 self.assertEqual(uri.params, {})
222
223
224 uri.scheme = test['scheme']
225 self.assertEqual(uri.scheme, test['scheme'])
226
227 uri.userinfo = test['userinfo']
228 self.assertEqual(uri.userinfo, test['userinfo'])
229 self.assertEqual(uri.username, test['username'])
230 self.assertEqual(uri.password, test['password'])
231
232 uri.hostname = test['hostname']
233 self.assertEqual(uri.hostname, test['hostname'])
234 self.assertEqual(uri.hostport, test['hostname'])
235
236 uri.port = test['port']
237 self.assertEqual(uri.port, test['port'])
238 self.assertEqual(uri.hostport, test['hostport'])
239
240 uri.path = test['path']
241 self.assertEqual(uri.path, test['path'])
242
243 uri.params = test['params']
244 self.assertEqual(uri.params, test['params'])
245
246 self.assertEqual(str(uri)+str(uri.relative), str(test['uri'])+str(test['relative']))
247
248 self.assertEqual(str(uri), test['uri'])
249
250 uri.params = {}
251 self.assertEqual(uri.params, {})
252 self.assertEqual(str(uri), (str(uri).split(";"))[0])
253
254class FetcherTest(unittest.TestCase):
255
256 def setUp(self):
257 self.d = bb.data.init()
258 self.tempdir = tempfile.mkdtemp()
259 self.dldir = os.path.join(self.tempdir, "download")
260 os.mkdir(self.dldir)
261 self.d.setVar("DL_DIR", self.dldir)
262 self.unpackdir = os.path.join(self.tempdir, "unpacked")
263 os.mkdir(self.unpackdir)
264 persistdir = os.path.join(self.tempdir, "persistdata")
265 self.d.setVar("PERSISTENT_DIR", persistdir)
266
267 def tearDown(self):
268 bb.utils.prunedir(self.tempdir)
269
270class MirrorUriTest(FetcherTest):
271
272 replaceuris = {
273 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/")
274 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
275 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
276 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
277 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
278 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
279 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http")
280 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
281 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake")
282 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890",
283 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache")
284 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
285 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/")
286 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
287 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3")
288 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
289 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz")
290 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
291 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist")
292 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2",
293 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/")
294 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2",
295 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
296 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
297 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
298 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
299 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http")
300 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
301
302 #Renaming files doesn't work
303 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
304 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
305 }
306
307 mirrorvar = "http://.*/.* file:///somepath/downloads/ \n" \
308 "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n" \
309 "https://.*/.* file:///someotherpath/downloads/ \n" \
310 "http://.*/.* file:///someotherpath/downloads/ \n"
311
312 def test_urireplace(self):
313 for k, v in self.replaceuris.items():
314 ud = bb.fetch.FetchData(k[0], self.d)
315 ud.setup_localpath(self.d)
316 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2]))
317 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d)
318 self.assertEqual([v], newuris)
319
320 def test_urilist1(self):
321 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
322 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
323 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
324 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
325
326 def test_urilist2(self):
327 # Catch https:// -> files:// bug
328 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
329 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
330 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
331 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
332
333class FetcherNetworkTest(FetcherTest):
334
335 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
336 print("Unset BB_SKIP_NETTESTS to run network tests")
337 else:
338 def test_fetch(self):
339 fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
340 fetcher.download()
341 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
342 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892)
343 self.d.setVar("BB_NO_NETWORK", "1")
344 fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
345 fetcher.download()
346 fetcher.unpack(self.unpackdir)
347 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9)
348 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9)
349
350 def test_fetch_mirror(self):
351 self.d.setVar("MIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
352 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
353 fetcher.download()
354 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
355
356 def test_fetch_premirror(self):
357 self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
358 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
359 fetcher.download()
360 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
361
362 def gitfetcher(self, url1, url2):
363 def checkrevision(self, fetcher):
364 fetcher.unpack(self.unpackdir)
365 revision = bb.process.run("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git")[0].strip()
366 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
367
368 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
369 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
370 fetcher = bb.fetch.Fetch([url1], self.d)
371 fetcher.download()
372 checkrevision(self, fetcher)
373 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works
374 bb.utils.prunedir(self.dldir + "/git2/")
375 bb.utils.prunedir(self.unpackdir)
376 self.d.setVar("BB_NO_NETWORK", "1")
377 fetcher = bb.fetch.Fetch([url2], self.d)
378 fetcher.download()
379 checkrevision(self, fetcher)
380
381 def test_gitfetch(self):
382 url1 = url2 = "git://git.openembedded.org/bitbake"
383 self.gitfetcher(url1, url2)
384
385 def test_gitfetch_premirror(self):
386 url1 = "git://git.openembedded.org/bitbake"
387 url2 = "git://someserver.org/bitbake"
388 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
389 self.gitfetcher(url1, url2)
390
391 def test_gitfetch_premirror2(self):
392 url1 = url2 = "git://someserver.org/bitbake"
393 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
394 self.gitfetcher(url1, url2)
395
396 def test_gitfetch_premirror3(self):
397 realurl = "git://git.openembedded.org/bitbake"
398 dummyurl = "git://someserver.org/bitbake"
399 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git")
400 os.chdir(self.tempdir)
401 bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True)
402 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (dummyurl, self.sourcedir))
403 self.gitfetcher(dummyurl, dummyurl)
404
405class URLHandle(unittest.TestCase):
406
407 datatable = {
408 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}),
409 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}),
410 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'})
411 }
412
413 def test_decodeurl(self):
414 for k, v in self.datatable.items():
415 result = bb.fetch.decodeurl(k)
416 self.assertEqual(result, v)
417
418 def test_encodeurl(self):
419 for k, v in self.datatable.items():
420 result = bb.fetch.encodeurl(v)
421 self.assertEqual(result, k)
422
423
424
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2024-04-18 11:55:16 +0000