summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/tests/fetch.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/tests/fetch.py')
-rw-r--r--bitbake/lib/bb/tests/fetch.py3799
1 files changed, 0 insertions, 3799 deletions
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
deleted file mode 100644
index e027705bf4..0000000000
--- a/bitbake/lib/bb/tests/fetch.py
+++ /dev/null
@@ -1,3799 +0,0 @@
1#
2# BitBake Tests for the Fetcher (fetch2/)
3#
4# Copyright (C) 2012 Richard Purdie
5#
6# SPDX-License-Identifier: GPL-2.0-only
7#
8
9import contextlib
10import shutil
11import unittest
12import unittest.mock
13import urllib.parse
14import hashlib
15import tempfile
16import collections
17import os
18import signal
19import tarfile
20from bb.fetch2 import URI
21from bb.fetch2 import FetchMethod
22import bb
23import bb.utils
24from bb.tests.support.httpserver import HTTPService
25
26def skipIfNoNetwork():
27 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
28 return unittest.skip("network test")
29 return lambda f: f
30
31
32@contextlib.contextmanager
33def hide_directory(directory):
34 """Hide the given directory and restore it after the context is left"""
35 temp_name = directory + ".bak"
36 os.rename(directory, temp_name)
37 try:
38 yield
39 finally:
40 os.rename(temp_name, directory)
41
42
43class TestTimeout(Exception):
44 # Indicate to pytest that this is not a test suite
45 __test__ = False
46
47class Timeout():
48
49 def __init__(self, seconds):
50 self.seconds = seconds
51
52 def handle_timeout(self, signum, frame):
53 raise TestTimeout("Test failed: timeout reached")
54
55 def __enter__(self):
56 signal.signal(signal.SIGALRM, self.handle_timeout)
57 signal.alarm(self.seconds)
58
59 def __exit__(self, exc_type, exc_val, exc_tb):
60 signal.alarm(0)
61
62class URITest(unittest.TestCase):
63 test_uris = {
64 "http://www.google.com/index.html" : {
65 'uri': 'http://www.google.com/index.html',
66 'scheme': 'http',
67 'hostname': 'www.google.com',
68 'port': None,
69 'hostport': 'www.google.com',
70 'path': '/index.html',
71 'userinfo': '',
72 'username': '',
73 'password': '',
74 'params': {},
75 'query': {},
76 'relative': False
77 },
78 "http://www.google.com/index.html;param1=value1" : {
79 'uri': 'http://www.google.com/index.html;param1=value1',
80 'scheme': 'http',
81 'hostname': 'www.google.com',
82 'port': None,
83 'hostport': 'www.google.com',
84 'path': '/index.html',
85 'userinfo': '',
86 'username': '',
87 'password': '',
88 'params': {
89 'param1': 'value1'
90 },
91 'query': {},
92 'relative': False
93 },
94 "http://www.example.org/index.html?param1=value1" : {
95 'uri': 'http://www.example.org/index.html?param1=value1',
96 'scheme': 'http',
97 'hostname': 'www.example.org',
98 'port': None,
99 'hostport': 'www.example.org',
100 'path': '/index.html',
101 'userinfo': '',
102 'username': '',
103 'password': '',
104 'params': {},
105 'query': {
106 'param1': 'value1'
107 },
108 'relative': False
109 },
110 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : {
111 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2',
112 'scheme': 'http',
113 'hostname': 'www.example.org',
114 'port': None,
115 'hostport': 'www.example.org',
116 'path': '/index.html',
117 'userinfo': '',
118 'username': '',
119 'password': '',
120 'params': {
121 'param2': 'value2'
122 },
123 'query': {
124 'qparam1': 'qvalue1'
125 },
126 'relative': False
127 },
128 # Check that trailing semicolons are handled correctly
129 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2;" : {
130 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2',
131 'scheme': 'http',
132 'hostname': 'www.example.org',
133 'port': None,
134 'hostport': 'www.example.org',
135 'path': '/index.html',
136 'userinfo': '',
137 'username': '',
138 'password': '',
139 'params': {
140 'param2': 'value2'
141 },
142 'query': {
143 'qparam1': 'qvalue1'
144 },
145 'relative': False
146 },
147 "http://www.example.com:8080/index.html" : {
148 'uri': 'http://www.example.com:8080/index.html',
149 'scheme': 'http',
150 'hostname': 'www.example.com',
151 'port': 8080,
152 'hostport': 'www.example.com:8080',
153 'path': '/index.html',
154 'userinfo': '',
155 'username': '',
156 'password': '',
157 'params': {},
158 'query': {},
159 'relative': False
160 },
161 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : {
162 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg',
163 'scheme': 'cvs',
164 'hostname': 'cvs.handhelds.org',
165 'port': None,
166 'hostport': 'cvs.handhelds.org',
167 'path': '/cvs',
168 'userinfo': 'anoncvs',
169 'username': 'anoncvs',
170 'password': '',
171 'params': {
172 'module': 'familiar/dist/ipkg'
173 },
174 'query': {},
175 'relative': False
176 },
177 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": {
178 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg',
179 'scheme': 'cvs',
180 'hostname': 'cvs.handhelds.org',
181 'port': None,
182 'hostport': 'cvs.handhelds.org',
183 'path': '/cvs',
184 'userinfo': 'anoncvs:anonymous',
185 'username': 'anoncvs',
186 'password': 'anonymous',
187 'params': collections.OrderedDict([
188 ('tag', 'V0-99-81'),
189 ('module', 'familiar/dist/ipkg')
190 ]),
191 'query': {},
192 'relative': False
193 },
194 "file://example.diff": { # NOTE: Not RFC compliant!
195 'uri': 'file:example.diff',
196 'scheme': 'file',
197 'hostname': '',
198 'port': None,
199 'hostport': '',
200 'path': 'example.diff',
201 'userinfo': '',
202 'username': '',
203 'password': '',
204 'params': {},
205 'query': {},
206 'relative': True
207 },
208 "file:example.diff": { # NOTE: RFC compliant version of the former
209 'uri': 'file:example.diff',
210 'scheme': 'file',
211 'hostname': '',
212 'port': None,
213 'hostport': '',
214 'path': 'example.diff',
215 'userinfo': '',
216 'userinfo': '',
217 'username': '',
218 'password': '',
219 'params': {},
220 'query': {},
221 'relative': True
222 },
223 "file:///tmp/example.diff": {
224 'uri': 'file:///tmp/example.diff',
225 'scheme': 'file',
226 'hostname': '',
227 'port': None,
228 'hostport': '',
229 'path': '/tmp/example.diff',
230 'userinfo': '',
231 'userinfo': '',
232 'username': '',
233 'password': '',
234 'params': {},
235 'query': {},
236 'relative': False
237 },
238 "git:///path/example.git": {
239 'uri': 'git:///path/example.git',
240 'scheme': 'git',
241 'hostname': '',
242 'port': None,
243 'hostport': '',
244 'path': '/path/example.git',
245 'userinfo': '',
246 'userinfo': '',
247 'username': '',
248 'password': '',
249 'params': {},
250 'query': {},
251 'relative': False
252 },
253 "git:path/example.git": {
254 'uri': 'git:path/example.git',
255 'scheme': 'git',
256 'hostname': '',
257 'port': None,
258 'hostport': '',
259 'path': 'path/example.git',
260 'userinfo': '',
261 'userinfo': '',
262 'username': '',
263 'password': '',
264 'params': {},
265 'query': {},
266 'relative': True
267 },
268 "git://example.net/path/example.git": {
269 'uri': 'git://example.net/path/example.git',
270 'scheme': 'git',
271 'hostname': 'example.net',
272 'port': None,
273 'hostport': 'example.net',
274 'path': '/path/example.git',
275 'userinfo': '',
276 'userinfo': '',
277 'username': '',
278 'password': '',
279 'params': {},
280 'query': {},
281 'relative': False
282 },
283 "git://tfs-example.org:22/tfs/example%20path/example.git": {
284 'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git',
285 'scheme': 'git',
286 'hostname': 'tfs-example.org',
287 'port': 22,
288 'hostport': 'tfs-example.org:22',
289 'path': '/tfs/example path/example.git',
290 'userinfo': '',
291 'userinfo': '',
292 'username': '',
293 'password': '',
294 'params': {},
295 'query': {},
296 'relative': False
297 },
298 "http://somesite.net;someparam=1": {
299 'uri': 'http://somesite.net;someparam=1',
300 'scheme': 'http',
301 'hostname': 'somesite.net',
302 'port': None,
303 'hostport': 'somesite.net',
304 'path': '',
305 'userinfo': '',
306 'userinfo': '',
307 'username': '',
308 'password': '',
309 'params': {"someparam" : "1"},
310 'query': {},
311 'relative': False
312 },
313 "file://somelocation;someparam=1": {
314 'uri': 'file:somelocation;someparam=1',
315 'scheme': 'file',
316 'hostname': '',
317 'port': None,
318 'hostport': '',
319 'path': 'somelocation',
320 'userinfo': '',
321 'userinfo': '',
322 'username': '',
323 'password': '',
324 'params': {"someparam" : "1"},
325 'query': {},
326 'relative': True
327 },
328 "https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip": {
329 'uri': 'https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip',
330 'scheme': 'https',
331 'hostname': 'www.innodisk.com',
332 'port': None,
333 'hostport': 'www.innodisk.com',
334 'path': '/Download_file',
335 'userinfo': '',
336 'userinfo': '',
337 'username': '',
338 'password': '',
339 'params': {"downloadfilename" : "EGPL-T101.zip"},
340 'query': {"9BE0BF6657": None},
341 'relative': False
342 },
343 "file://example@.service": {
344 'uri': 'file:example%40.service',
345 'scheme': 'file',
346 'hostname': '',
347 'port': None,
348 'hostport': '',
349 'path': 'example@.service',
350 'userinfo': '',
351 'userinfo': '',
352 'username': '',
353 'password': '',
354 'params': {},
355 'query': {},
356 'relative': True
357 }
358
359 }
360
361 def test_uri(self):
362 for test_uri, ref in self.test_uris.items():
363 uri = URI(test_uri)
364
365 self.assertEqual(str(uri), ref['uri'])
366
367 # expected attributes
368 self.assertEqual(uri.scheme, ref['scheme'])
369
370 self.assertEqual(uri.userinfo, ref['userinfo'])
371 self.assertEqual(uri.username, ref['username'])
372 self.assertEqual(uri.password, ref['password'])
373
374 self.assertEqual(uri.hostname, ref['hostname'])
375 self.assertEqual(uri.port, ref['port'])
376 self.assertEqual(uri.hostport, ref['hostport'])
377
378 self.assertEqual(uri.path, ref['path'])
379 self.assertEqual(uri.params, ref['params'])
380
381 self.assertEqual(uri.relative, ref['relative'])
382
383 def test_dict(self):
384 for test in self.test_uris.values():
385 uri = URI()
386
387 self.assertEqual(uri.scheme, '')
388 self.assertEqual(uri.userinfo, '')
389 self.assertEqual(uri.username, '')
390 self.assertEqual(uri.password, '')
391 self.assertEqual(uri.hostname, '')
392 self.assertEqual(uri.port, None)
393 self.assertEqual(uri.path, '')
394 self.assertEqual(uri.params, {})
395
396
397 uri.scheme = test['scheme']
398 self.assertEqual(uri.scheme, test['scheme'])
399
400 uri.userinfo = test['userinfo']
401 self.assertEqual(uri.userinfo, test['userinfo'])
402 self.assertEqual(uri.username, test['username'])
403 self.assertEqual(uri.password, test['password'])
404
405 # make sure changing the values doesn't do anything unexpected
406 uri.username = 'changeme'
407 self.assertEqual(uri.username, 'changeme')
408 self.assertEqual(uri.password, test['password'])
409 uri.password = 'insecure'
410 self.assertEqual(uri.username, 'changeme')
411 self.assertEqual(uri.password, 'insecure')
412
413 # reset back after our trickery
414 uri.userinfo = test['userinfo']
415 self.assertEqual(uri.userinfo, test['userinfo'])
416 self.assertEqual(uri.username, test['username'])
417 self.assertEqual(uri.password, test['password'])
418
419 uri.hostname = test['hostname']
420 self.assertEqual(uri.hostname, test['hostname'])
421 self.assertEqual(uri.hostport, test['hostname'])
422
423 uri.port = test['port']
424 self.assertEqual(uri.port, test['port'])
425 self.assertEqual(uri.hostport, test['hostport'])
426
427 uri.path = test['path']
428 self.assertEqual(uri.path, test['path'])
429
430 uri.params = test['params']
431 self.assertEqual(uri.params, test['params'])
432
433 uri.query = test['query']
434 self.assertEqual(uri.query, test['query'])
435
436 self.assertEqual(str(uri), test['uri'])
437
438 uri.params = {}
439 self.assertEqual(uri.params, {})
440 self.assertEqual(str(uri), (str(uri).split(";"))[0])
441
442class FetcherTest(unittest.TestCase):
443
444 def setUp(self):
445 self.origdir = os.getcwd()
446 self.d = bb.data.init()
447 self.tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-")
448 self.dldir = os.path.join(self.tempdir, "download")
449 os.mkdir(self.dldir)
450 self.d.setVar("DL_DIR", self.dldir)
451 self.unpackdir = os.path.join(self.tempdir, "unpacked")
452 os.mkdir(self.unpackdir)
453 persistdir = os.path.join(self.tempdir, "persistdata")
454 self.d.setVar("PERSISTENT_DIR", persistdir)
455
456 def tearDown(self):
457 os.chdir(self.origdir)
458 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
459 print("Not cleaning up %s. Please remove manually." % self.tempdir)
460 else:
461 bb.process.run('chmod u+rw -R %s' % self.tempdir)
462 bb.utils.prunedir(self.tempdir)
463
464 def git(self, cmd, cwd=None):
465 if isinstance(cmd, str):
466 cmd = 'git -c safe.bareRepository=all ' + cmd
467 else:
468 cmd = ['git', '-c', 'safe.bareRepository=all'] + cmd
469 if cwd is None:
470 cwd = self.gitdir
471 return bb.process.run(cmd, cwd=cwd)[0]
472
473 def git_init(self, cwd=None):
474 self.git('init', cwd=cwd)
475 # Explicitly set initial branch to master as
476 # a common setup is to use other default
477 # branch than master.
478 self.git(['checkout', '-b', 'master'], cwd=cwd)
479
480 try:
481 self.git(['config', 'user.email'], cwd=cwd)
482 except bb.process.ExecutionError:
483 self.git(['config', 'user.email', 'you@example.com'], cwd=cwd)
484
485 try:
486 self.git(['config', 'user.name'], cwd=cwd)
487 except bb.process.ExecutionError:
488 self.git(['config', 'user.name', 'Your Name'], cwd=cwd)
489
490class MirrorUriTest(FetcherTest):
491
492 replaceuris = {
493 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/.*", "http://somewhere.org/somedir/")
494 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
495 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
496 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
497 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
498 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
499 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http")
500 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
501 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake")
502 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master",
503 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache")
504 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
505 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/")
506 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
507 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3")
508 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
509 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz")
510 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
511 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist")
512 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2",
513 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/")
514 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2",
515 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
516 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
517 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
518 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
519 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http")
520 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
521 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org")
522 : "http://somewhere2.org/somefile_1.2.3.tar.gz",
523 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/")
524 : "http://somewhere2.org/somefile_1.2.3.tar.gz",
525 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http")
526 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
527 ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http")
528 : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
529 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=git;branch=master", "git://someserver.org/bitbake", "git://someotherserver.org/bitbake;protocol=https")
530 : "git://someotherserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=https;branch=master",
531 ("gitsm://git.qemu.org/git/seabios.git/;protocol=https;name=roms/seabios;subpath=roms/seabios;bareclone=1;nobranch=1;rev=1234567890123456789012345678901234567890", "gitsm://.*/.*", "http://petalinux.xilinx.com/sswreleases/rel-v${XILINX_VER_MAIN}/downloads") : "http://petalinux.xilinx.com/sswreleases/rel-v%24%7BXILINX_VER_MAIN%7D/downloads/git2_git.qemu.org.git.seabios.git..tar.gz",
532 ("https://somewhere.org/example/1.0.0/example;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/PATH")
533 : "file:///mirror/example/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz",
534 ("https://somewhere.org/example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/some-example-1.0.0.tgz")
535 : "file:///mirror/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz",
536 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", r"git://(?!internal\.git\.server).*/.*", "http://somewhere.org/somedir/")
537 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
538 ("git://internal.git.server.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", r"git://(?!internal\.git\.server).*/.*", "http://somewhere.org/somedir/")
539 : None,
540
541 #Renaming files doesn't work
542 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
543 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
544 }
545
546 mirrorvar = "http://.*/.* file:///somepath/downloads/ " \
547 "git://someserver.org/bitbake git://git.openembedded.org/bitbake " \
548 "https?://.*/.* file:///someotherpath/downloads/ " \
549 "svn://svn.server1.com/ svn://svn.server2.com/"
550
551 def test_urireplace(self):
552 self.d.setVar("FILESPATH", ".")
553 for k, v in self.replaceuris.items():
554 ud = bb.fetch.FetchData(k[0], self.d)
555 ud.setup_localpath(self.d)
556 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2]))
557 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d)
558 self.assertEqual([v] if v else [], newuris)
559
560 def test_urilist1(self):
561 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
562 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
563 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
564 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
565
566 def test_urilist2(self):
567 # Catch https:// -> files:// bug
568 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
569 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
570 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
571 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
572
573 def test_urilistsvn(self):
574 # Catch svn:// -> svn:// bug
575 fetcher = bb.fetch.FetchData("svn://svn.server1.com/isource/svnroot/reponame/tags/tagname;module=path_in_tagnamefolder;protocol=https;rev=2", self.d)
576 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
577 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
578 self.assertEqual(uris, ['svn://svn.server2.com/isource/svnroot/reponame/tags/tagname;module=path_in_tagnamefolder;protocol=https;rev=2'])
579
580 def test_mirror_of_mirror(self):
581 # Test if mirror of a mirror works
582 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/"
583 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/"
584 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
585 mirrors = bb.fetch2.mirror_from_string(mirrorvar)
586 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
587 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz',
588 'file:///someotherpath/downloads/bitbake-1.0.tar.gz',
589 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz',
590 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz'])
591
592 recmirrorvar = "https://.*/[^/]* http://aaaa/A/A/A/ " \
593 "https://.*/[^/]* https://bbbb/B/B/B/"
594
595 def test_recursive(self):
596 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
597 mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar)
598 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
599 self.assertEqual(uris, ['http://aaaa/A/A/A/bitbake/bitbake-1.0.tar.gz',
600 'https://bbbb/B/B/B/bitbake/bitbake-1.0.tar.gz',
601 'http://aaaa/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz'])
602
603
604class GitDownloadDirectoryNamingTest(FetcherTest):
605 def setUp(self):
606 super(GitDownloadDirectoryNamingTest, self).setUp()
607 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
608 self.recipe_dir = "git.openembedded.org.bitbake"
609 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
610 self.mirror_dir = "github.com.openembedded.bitbake.git"
611
612 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
613
614 def setup_mirror_rewrite(self):
615 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url)
616
617 @skipIfNoNetwork()
618 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self):
619 self.setup_mirror_rewrite()
620 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
621
622 fetcher.download()
623
624 dir = os.listdir(self.dldir + "/git2")
625 self.assertIn(self.recipe_dir, dir)
626
627 @skipIfNoNetwork()
628 def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self):
629 self.setup_mirror_rewrite()
630 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
631
632 fetcher.download()
633
634 dir = os.listdir(self.dldir + "/git2")
635 self.assertIn(self.mirror_dir, dir)
636 self.assertIn(self.recipe_dir, dir)
637
638 @skipIfNoNetwork()
639 def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self):
640 self.setup_mirror_rewrite()
641 fetcher = bb.fetch.Fetch([self.mirror_url], self.d)
642 fetcher.download()
643 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
644
645 fetcher.download()
646
647 dir = os.listdir(self.dldir + "/git2")
648 self.assertIn(self.mirror_dir, dir)
649 self.assertIn(self.recipe_dir, dir)
650
651
652class TarballNamingTest(FetcherTest):
653 def setUp(self):
654 super(TarballNamingTest, self).setUp()
655 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
656 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
657 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
658 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz"
659
660 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
661 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
662
663 def setup_mirror_rewrite(self):
664 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url)
665
666 @skipIfNoNetwork()
667 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self):
668 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
669
670 fetcher.download()
671
672 dir = os.listdir(self.dldir)
673 self.assertIn(self.recipe_tarball, dir)
674
675 @skipIfNoNetwork()
676 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
677 self.setup_mirror_rewrite()
678 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
679
680 fetcher.download()
681
682 dir = os.listdir(self.dldir)
683 self.assertIn(self.mirror_tarball, dir)
684
685
686class GitShallowTarballNamingTest(FetcherTest):
687 def setUp(self):
688 super(GitShallowTarballNamingTest, self).setUp()
689 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
690 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz"
691 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
692 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz"
693
694 self.d.setVar('BB_GIT_SHALLOW', '1')
695 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
696 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
697
698 def setup_mirror_rewrite(self):
699 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url)
700
701 @skipIfNoNetwork()
702 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self):
703 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
704
705 fetcher.download()
706
707 dir = os.listdir(self.dldir)
708 self.assertIn(self.recipe_tarball, dir)
709
710 @skipIfNoNetwork()
711 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
712 self.setup_mirror_rewrite()
713 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
714
715 fetcher.download()
716
717 dir = os.listdir(self.dldir)
718 self.assertIn(self.mirror_tarball, dir)
719
720
721class CleanTarballTest(FetcherTest):
722 def setUp(self):
723 super(CleanTarballTest, self).setUp()
724 self.recipe_url = "git://git.openembedded.org/bitbake;protocol=https;branch=master"
725 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
726
727 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
728 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
729
730 @skipIfNoNetwork()
731 def test_that_the_tarball_contents_does_not_leak_info(self):
732 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
733
734 fetcher.download()
735
736 fetcher.unpack(self.unpackdir)
737 mtime = bb.process.run('git log --all -1 --format=%ct',
738 cwd=os.path.join(self.unpackdir, 'git'))
739 self.assertEqual(len(mtime), 2)
740 mtime = int(mtime[0])
741
742 archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball))
743 self.assertNotEqual(len(archive.members), 0)
744 for member in archive.members:
745 if member.name == ".":
746 continue
747 self.assertEqual(member.uname, 'oe', "user name for %s differs" % member.name)
748 self.assertEqual(member.uid, 0, "uid for %s differs" % member.name)
749 self.assertEqual(member.gname, 'oe', "group name for %s differs" % member.name)
750 self.assertEqual(member.gid, 0, "gid for %s differs" % member.name)
751 self.assertEqual(member.mtime, mtime, "mtime for %s differs" % member.name)
752
753
754class FetcherLocalTest(FetcherTest):
755 def setUp(self):
756 def touch(fn):
757 with open(fn, 'a'):
758 os.utime(fn, None)
759
760 super(FetcherLocalTest, self).setUp()
761 self.localsrcdir = os.path.join(self.tempdir, 'localsrc')
762 os.makedirs(self.localsrcdir)
763 touch(os.path.join(self.localsrcdir, 'a'))
764 touch(os.path.join(self.localsrcdir, 'b'))
765 touch(os.path.join(self.localsrcdir, 'c@d'))
766 os.makedirs(os.path.join(self.localsrcdir, 'dir'))
767 touch(os.path.join(self.localsrcdir, 'dir', 'c'))
768 touch(os.path.join(self.localsrcdir, 'dir', 'd'))
769 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir'))
770 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e'))
771 touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device'))
772 bb.process.run('tar cf archive.tar -C dir .', cwd=self.localsrcdir)
773 bb.process.run('tar czf archive.tar.gz -C dir .', cwd=self.localsrcdir)
774 bb.process.run('tar cjf archive.tar.bz2 -C dir .', cwd=self.localsrcdir)
775 self.d.setVar("FILESPATH", self.localsrcdir)
776
777 def fetchUnpack(self, uris):
778 fetcher = bb.fetch.Fetch(uris, self.d)
779 fetcher.download()
780 fetcher.unpack(self.unpackdir)
781 flst = []
782 for root, dirs, files in os.walk(self.unpackdir):
783 for f in files:
784 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir))
785 flst.sort()
786 return flst
787
788 def test_local_checksum_fails_no_file(self):
789 self.d.setVar("SRC_URI", "file://404")
790 with self.assertRaises(bb.BBHandledException):
791 bb.fetch.get_checksum_file_list(self.d)
792
793 def test_local(self):
794 tree = self.fetchUnpack(['file://a', 'file://dir/c'])
795 self.assertEqual(tree, ['a', 'dir/c'])
796
797 def test_local_at(self):
798 tree = self.fetchUnpack(['file://c@d'])
799 self.assertEqual(tree, ['c@d'])
800
801 def test_local_backslash(self):
802 tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device'])
803 self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device'])
804
805 def test_local_wildcard(self):
806 with self.assertRaises(bb.fetch2.ParameterError):
807 tree = self.fetchUnpack(['file://a', 'file://dir/*'])
808
809 def test_local_dir(self):
810 tree = self.fetchUnpack(['file://a', 'file://dir'])
811 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e'])
812
813 def test_local_subdir(self):
814 tree = self.fetchUnpack(['file://dir/subdir'])
815 self.assertEqual(tree, ['dir/subdir/e'])
816
817 def test_local_subdir_file(self):
818 tree = self.fetchUnpack(['file://dir/subdir/e'])
819 self.assertEqual(tree, ['dir/subdir/e'])
820
821 def test_local_subdirparam(self):
822 tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo'])
823 self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e'])
824
825 def test_local_deepsubdirparam(self):
826 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar'])
827 self.assertEqual(tree, ['bar/dir/subdir/e'])
828
829 def test_local_absolutedir(self):
830 # Unpacking to an absolute path that is a subdirectory of the root
831 # should work
832 tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')])
833
834 # Unpacking to an absolute path outside of the root should fail
835 with self.assertRaises(bb.fetch2.UnpackError):
836 self.fetchUnpack(['file://a;subdir=/bin/sh'])
837
838 def test_local_striplevel(self):
839 tree = self.fetchUnpack(['file://archive.tar;subdir=bar;striplevel=1'])
840 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e'])
841
842 def test_local_striplevel_gzip(self):
843 tree = self.fetchUnpack(['file://archive.tar.gz;subdir=bar;striplevel=1'])
844 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e'])
845
846 def test_local_striplevel_bzip2(self):
847 tree = self.fetchUnpack(['file://archive.tar.bz2;subdir=bar;striplevel=1'])
848 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e'])
849
850 def dummyGitTest(self, suffix):
851 # Create dummy local Git repo
852 src_dir = tempfile.mkdtemp(dir=self.tempdir,
853 prefix='gitfetch_localusehead_')
854 self.gitdir = os.path.abspath(src_dir)
855 self.git_init()
856 self.git(['commit', '--allow-empty', '-m', 'Dummy commit'])
857 # Use other branch than master
858 self.git(['checkout', '-b', 'my-devel'])
859 self.git(['commit', '--allow-empty', '-m', 'Dummy commit 2'])
860 orig_rev = self.git(['rev-parse', 'HEAD']).strip()
861
862 # Fetch and check revision
863 self.d.setVar("SRCREV", "AUTOINC")
864 self.d.setVar("__BBSRCREV_SEEN", "1")
865 url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix
866 fetcher = bb.fetch.Fetch([url], self.d)
867 fetcher.download()
868 fetcher.unpack(self.unpackdir)
869 unpack_rev = self.git(['rev-parse', 'HEAD'],
870 cwd=os.path.join(self.unpackdir, 'git')).strip()
871 self.assertEqual(orig_rev, unpack_rev)
872
873 def test_local_gitfetch_usehead(self):
874 self.dummyGitTest("usehead=1")
875
876 def test_local_gitfetch_usehead_withname(self):
877 self.dummyGitTest("usehead=1;name=newName")
878
879 def test_local_gitfetch_shared(self):
880 self.dummyGitTest("usehead=1;name=sharedName")
881 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
882 self.assertTrue(os.path.exists(alt))
883
884 def test_local_gitfetch_noshared(self):
885 self.d.setVar('BB_GIT_NOSHARED', '1')
886 self.unpackdir += '_noshared'
887 self.dummyGitTest("usehead=1;name=noSharedName")
888 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
889 self.assertFalse(os.path.exists(alt))
890
891class FetcherNoNetworkTest(FetcherTest):
892 def setUp(self):
893 super().setUp()
894 # all test cases are based on not having network
895 self.d.setVar("BB_NO_NETWORK", "1")
896
897 def test_missing(self):
898 string = "this is a test file\n".encode("utf-8")
899 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
900 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
901
902 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
903 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
904 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
905 with self.assertRaises(bb.fetch2.NetworkAccess):
906 fetcher.download()
907
908 def test_valid_missing_donestamp(self):
909 # create the file in the download directory with correct hash
910 string = "this is a test file\n".encode("utf-8")
911 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f:
912 f.write(string)
913
914 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
915 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
916
917 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
918 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
919 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
920 fetcher.download()
921 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
922
923 def test_invalid_missing_donestamp(self):
924 # create an invalid file in the download directory with incorrect hash
925 string = "this is a test file\n".encode("utf-8")
926 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
927 pass
928
929 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
930 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
931
932 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
933 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
934 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
935 with self.assertRaises(bb.fetch2.NetworkAccess):
936 fetcher.download()
937 # the existing file should not exist or should have be moved to "bad-checksum"
938 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
939
940 def test_nochecksums_missing(self):
941 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
942 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
943 # ssh fetch does not support checksums
944 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
945 # attempts to download with missing donestamp
946 with self.assertRaises(bb.fetch2.NetworkAccess):
947 fetcher.download()
948
949 def test_nochecksums_missing_donestamp(self):
950 # create a file in the download directory
951 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
952 pass
953
954 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
955 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
956 # ssh fetch does not support checksums
957 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
958 # attempts to download with missing donestamp
959 with self.assertRaises(bb.fetch2.NetworkAccess):
960 fetcher.download()
961
962 def test_nochecksums_has_donestamp(self):
963 # create a file in the download directory with the donestamp
964 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
965 pass
966 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
967 pass
968
969 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
970 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
971 # ssh fetch does not support checksums
972 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
973 # should not fetch
974 fetcher.download()
975 # both files should still exist
976 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
977 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
978
979 def test_nochecksums_missing_has_donestamp(self):
980 # create a file in the download directory with the donestamp
981 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
982 pass
983
984 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
985 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
986 # ssh fetch does not support checksums
987 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
988 with self.assertRaises(bb.fetch2.NetworkAccess):
989 fetcher.download()
990 # both files should still exist
991 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
992 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
993
994class FetcherNetworkTest(FetcherTest):
995 @skipIfNoNetwork()
996 def test_fetch(self):
997 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
998 fetcher.download()
999 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
1000 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892)
1001 self.d.setVar("BB_NO_NETWORK", "1")
1002 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
1003 fetcher.download()
1004 fetcher.unpack(self.unpackdir)
1005 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9)
1006 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9)
1007
1008 @skipIfNoNetwork()
1009 def test_fetch_mirror(self):
1010 self.d.setVar("MIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
1011 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
1012 fetcher.download()
1013 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
1014
1015 @skipIfNoNetwork()
1016 def test_fetch_mirror_of_mirror(self):
1017 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake")
1018 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
1019 fetcher.download()
1020 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
1021
1022 @skipIfNoNetwork()
1023 def test_fetch_file_mirror_of_mirror(self):
1024 self.d.setVar("FILESPATH", ".")
1025 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake")
1026 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
1027 os.mkdir(self.dldir + "/some2where")
1028 fetcher.download()
1029 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
1030
1031 @skipIfNoNetwork()
1032 def test_fetch_premirror(self):
1033 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
1034 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
1035 fetcher.download()
1036 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
1037
1038 @skipIfNoNetwork()
1039 def test_fetch_specify_downloadfilename(self):
1040 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d)
1041 fetcher.download()
1042 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749)
1043
1044 @skipIfNoNetwork()
1045 def test_fetch_premirror_specify_downloadfilename_regex_uri(self):
1046 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/")
1047 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d)
1048 fetcher.download()
1049 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
1050
1051 @skipIfNoNetwork()
1052 # BZ13039
1053 def test_fetch_premirror_specify_downloadfilename_specific_uri(self):
1054 self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake")
1055 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d)
1056 fetcher.download()
1057 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
1058
1059 @skipIfNoNetwork()
1060 def test_fetch_premirror_use_downloadfilename_to_fetch(self):
1061 # Ensure downloadfilename is used when fetching from premirror.
1062 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
1063 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d)
1064 fetcher.download()
1065 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
1066
1067 @skipIfNoNetwork()
1068 def gitfetcher(self, url1, url2):
1069 def checkrevision(self, fetcher):
1070 fetcher.unpack(self.unpackdir)
1071 revision = self.git(['rev-parse', 'HEAD'],
1072 cwd=os.path.join(self.unpackdir, 'git')).strip()
1073 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
1074
1075 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
1076 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
1077 fetcher = bb.fetch.Fetch([url1], self.d)
1078 fetcher.download()
1079 checkrevision(self, fetcher)
1080 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works
1081 bb.utils.prunedir(self.dldir + "/git2/")
1082 bb.utils.prunedir(self.unpackdir)
1083 self.d.setVar("BB_NO_NETWORK", "1")
1084 fetcher = bb.fetch.Fetch([url2], self.d)
1085 fetcher.download()
1086 checkrevision(self, fetcher)
1087
1088 @skipIfNoNetwork()
1089 def test_gitfetch(self):
1090 url1 = url2 = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
1091 self.gitfetcher(url1, url2)
1092
1093 @skipIfNoNetwork()
1094 def test_gitfetch_goodsrcrev(self):
1095 # SRCREV is set but matches rev= parameter
1096 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https"
1097 self.gitfetcher(url1, url2)
1098
1099 @skipIfNoNetwork()
1100 def test_gitfetch_badsrcrev(self):
1101 # SRCREV is set but does not match rev= parameter
1102 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https"
1103 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
1104
1105 @skipIfNoNetwork()
1106 def test_gitfetch_usehead(self):
1107 # Since self.gitfetcher() sets SRCREV we expect this to override
1108 # `usehead=1' and instead fetch the specified SRCREV. See
1109 # test_local_gitfetch_usehead() for a positive use of the usehead
1110 # feature.
1111 url = "git://git.openembedded.org/bitbake;usehead=1;branch=master;protocol=https"
1112 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
1113
1114 @skipIfNoNetwork()
1115 def test_gitfetch_usehead_withname(self):
1116 # Since self.gitfetcher() sets SRCREV we expect this to override
1117 # `usehead=1' and instead fetch the specified SRCREV. See
1118 # test_local_gitfetch_usehead() for a positive use of the usehead
1119 # feature.
1120 url = "git://git.openembedded.org/bitbake;usehead=1;name=newName;branch=master;protocol=https"
1121 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
1122
1123 @skipIfNoNetwork()
1124 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self):
1125 recipeurl = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
1126 mirrorurl = "git://someserver.org/bitbake;branch=master;protocol=https"
1127 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake")
1128 self.gitfetcher(recipeurl, mirrorurl)
1129
1130 @skipIfNoNetwork()
1131 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self):
1132 recipeurl = "git://someserver.org/bitbake;branch=master;protocol=https"
1133 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake")
1134 self.gitfetcher(recipeurl, recipeurl)
1135
1136 @skipIfNoNetwork()
1137 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self):
1138 realurl = "https://git.openembedded.org/bitbake"
1139 recipeurl = "git://someserver.org/bitbake;protocol=https;branch=master"
1140 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git")
1141 os.chdir(self.tempdir)
1142 self.git(['clone', realurl, self.sourcedir], cwd=self.tempdir)
1143 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipeurl, self.sourcedir))
1144 self.gitfetcher(recipeurl, recipeurl)
1145
1146 @skipIfNoNetwork()
1147 def test_git_submodule(self):
1148 # URL with ssh submodules
1149 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master;protocol=https"
1150 # Original URL (comment this if you have ssh access to git.yoctoproject.org)
1151 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master;protocol=https"
1152 fetcher = bb.fetch.Fetch([url], self.d)
1153 fetcher.download()
1154 # Previous cwd has been deleted
1155 os.chdir(os.path.dirname(self.unpackdir))
1156 fetcher.unpack(self.unpackdir)
1157
1158 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1159 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing')
1160 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing')
1161 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present')
1162
1163 # Only when we're running the extended test with a submodule's submodule, can we check this.
1164 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
1165 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
1166
1167 @skipIfNoNetwork()
1168 def test_git_submodule_restricted_network_premirrors(self):
1169 # this test is to ensure that premirrors will be tried in restricted network
1170 # that is, BB_ALLOWED_NETWORKS does not contain the domain the url uses
1171 url = "gitsm://github.com/grpc/grpc.git;protocol=https;name=grpc;branch=v1.60.x;rev=0ef13a7555dbaadd4633399242524129eef5e231"
1172 # create a download directory to be used as premirror later
1173 tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-")
1174 dl_premirror = os.path.join(tempdir, "download-premirror")
1175 os.mkdir(dl_premirror)
1176 self.d.setVar("DL_DIR", dl_premirror)
1177 fetcher = bb.fetch.Fetch([url], self.d)
1178 fetcher.download()
1179 # now use the premirror in restricted network
1180 self.d.setVar("DL_DIR", self.dldir)
1181 self.d.setVar("PREMIRRORS", "gitsm://.*/.* gitsm://%s/git2/MIRRORNAME;protocol=file" % dl_premirror)
1182 self.d.setVar("BB_ALLOWED_NETWORKS", "*.some.domain")
1183 fetcher = bb.fetch.Fetch([url], self.d)
1184 fetcher.download()
1185
1186 @skipIfNoNetwork()
1187 def test_git_submodule_dbus_broker(self):
1188 # The following external repositories have show failures in fetch and unpack operations
1189 # We want to avoid regressions!
1190 url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main"
1191 fetcher = bb.fetch.Fetch([url], self.d)
1192 fetcher.download()
1193 # Previous cwd has been deleted
1194 os.chdir(os.path.dirname(self.unpackdir))
1195 fetcher.unpack(self.unpackdir)
1196
1197 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1198 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"')
1199 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"')
1200 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"')
1201 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"')
1202 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"')
1203
1204 @skipIfNoNetwork()
1205 def test_git_submodule_CLI11(self):
1206 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main"
1207 fetcher = bb.fetch.Fetch([url], self.d)
1208 fetcher.download()
1209 # Previous cwd has been deleted
1210 os.chdir(os.path.dirname(self.unpackdir))
1211 fetcher.unpack(self.unpackdir)
1212
1213 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1214 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
1215 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
1216 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
1217
1218 @skipIfNoNetwork()
1219 def test_git_submodule_update_CLI11(self):
1220 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
1221 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main"
1222 fetcher = bb.fetch.Fetch([url], self.d)
1223 fetcher.download()
1224
1225 # CLI11 that pulls in a newer nlohmann-json
1226 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main"
1227 fetcher = bb.fetch.Fetch([url], self.d)
1228 fetcher.download()
1229 # Previous cwd has been deleted
1230 os.chdir(os.path.dirname(self.unpackdir))
1231 fetcher.unpack(self.unpackdir)
1232
1233 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1234 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
1235 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
1236 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
1237
1238 @skipIfNoNetwork()
1239 def test_git_submodule_aktualizr(self):
1240 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
1241 fetcher = bb.fetch.Fetch([url], self.d)
1242 fetcher.download()
1243 # Previous cwd has been deleted
1244 os.chdir(os.path.dirname(self.unpackdir))
1245 fetcher.unpack(self.unpackdir)
1246
1247 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1248 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"')
1249 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"')
1250 self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly")
1251 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"')
1252 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"')
1253 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"')
1254
1255 @skipIfNoNetwork()
1256 def test_git_submodule_iotedge(self):
1257 """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """
1258
1259 # This repository also has submodules where the module (name), path and url do not align
1260 url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699;branch=main"
1261 fetcher = bb.fetch.Fetch([url], self.d)
1262 fetcher.download()
1263 # Previous cwd has been deleted
1264 os.chdir(os.path.dirname(self.unpackdir))
1265 fetcher.unpack(self.unpackdir)
1266
1267 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1268
1269 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/README.md')), msg='Missing submodule checkout')
1270 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/ctest/README.md')), msg='Missing submodule checkout')
1271 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/testrunner/readme.md')), msg='Missing submodule checkout')
1272 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/readme.md')), msg='Missing submodule checkout')
1273 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
1274 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
1275 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/README.md')), msg='Missing submodule checkout')
1276 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/README.md')), msg='Missing submodule checkout')
1277 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/ctest/README.md')), msg='Missing submodule checkout')
1278 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/testrunner/readme.md')), msg='Missing submodule checkout')
1279 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/readme.md')), msg='Missing submodule checkout')
1280 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
1281 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
1282
1283 @skipIfNoNetwork()
1284 def test_git_submodule_reference_to_parent(self):
1285 self.recipe_url = "gitsm://github.com/gflags/gflags.git;protocol=https;branch=master"
1286 self.d.setVar("SRCREV", "14e1138441bbbb584160cb1c0a0426ec1bac35f1")
1287 with Timeout(60):
1288 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
1289 with self.assertRaises(bb.fetch2.FetchError):
1290 fetcher.download()
1291
1292class SVNTest(FetcherTest):
1293 def skipIfNoSvn():
1294 if not shutil.which("svn"):
1295 return unittest.skip("svn not installed, tests being skipped")
1296
1297 if not shutil.which("svnadmin"):
1298 return unittest.skip("svnadmin not installed, tests being skipped")
1299
1300 return lambda f: f
1301
1302 @skipIfNoSvn()
1303 def setUp(self):
1304 """ Create a local repository """
1305
1306 super(SVNTest, self).setUp()
1307
1308 # Create something we can fetch
1309 src_dir = tempfile.mkdtemp(dir=self.tempdir,
1310 prefix='svnfetch_srcdir_')
1311 src_dir = os.path.abspath(src_dir)
1312 bb.process.run("echo readme > README.md", cwd=src_dir)
1313
1314 # Store it in a local SVN repository
1315 repo_dir = tempfile.mkdtemp(dir=self.tempdir,
1316 prefix='svnfetch_localrepo_')
1317 repo_dir = os.path.abspath(repo_dir)
1318 bb.process.run("svnadmin create project", cwd=repo_dir)
1319
1320 self.repo_url = "file://%s/project" % repo_dir
1321 bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url),
1322 cwd=repo_dir)
1323
1324 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
1325 # Github won't emulate SVN anymore (see https://github.blog/2023-01-20-sunsetting-subversion-support/)
1326 # Use still accessible svn repo (only trunk to avoid longer downloads)
1327 bb.process.run("svn propset svn:externals 'bitbake https://svn.apache.org/repos/asf/serf/trunk' .",
1328 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
1329 bb.process.run("svn commit --non-interactive -m 'Add external'",
1330 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
1331
1332 self.src_dir = src_dir
1333 self.repo_dir = repo_dir
1334
1335 @skipIfNoSvn()
1336 def tearDown(self):
1337 os.chdir(self.origdir)
1338 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
1339 print("Not cleaning up %s. Please remove manually." % self.tempdir)
1340 else:
1341 bb.utils.prunedir(self.tempdir)
1342
1343 @skipIfNoSvn()
1344 @skipIfNoNetwork()
1345 def test_noexternal_svn(self):
1346 # Always match the rev count from setUp (currently rev 2)
1347 url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '')
1348 fetcher = bb.fetch.Fetch([url], self.d)
1349 fetcher.download()
1350 os.chdir(os.path.dirname(self.unpackdir))
1351 fetcher.unpack(self.unpackdir)
1352
1353 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
1354 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
1355 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should NOT exist")
1356 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should NOT exit")
1357
1358 @skipIfNoSvn()
1359 def test_external_svn(self):
1360 # Always match the rev count from setUp (currently rev 2)
1361 url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '')
1362 fetcher = bb.fetch.Fetch([url], self.d)
1363 fetcher.download()
1364 os.chdir(os.path.dirname(self.unpackdir))
1365 fetcher.unpack(self.unpackdir)
1366
1367 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
1368 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
1369 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should exist")
1370 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should exit")
1371
1372class TrustedNetworksTest(FetcherTest):
1373 def test_trusted_network(self):
1374 # Ensure trusted_network returns False when the host IS in the list.
1375 url = "git://Someserver.org/foo;rev=1;branch=master"
1376 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org")
1377 self.assertTrue(bb.fetch.trusted_network(self.d, url))
1378
1379 def test_wild_trusted_network(self):
1380 # Ensure trusted_network returns true when the *.host IS in the list.
1381 url = "git://Someserver.org/foo;rev=1;branch=master"
1382 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1383 self.assertTrue(bb.fetch.trusted_network(self.d, url))
1384
1385 def test_prefix_wild_trusted_network(self):
1386 # Ensure trusted_network returns true when the prefix matches *.host.
1387 url = "git://git.Someserver.org/foo;rev=1;branch=master"
1388 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1389 self.assertTrue(bb.fetch.trusted_network(self.d, url))
1390
1391 def test_two_prefix_wild_trusted_network(self):
1392 # Ensure trusted_network returns true when the prefix matches *.host.
1393 url = "git://something.git.Someserver.org/foo;rev=1;branch=master"
1394 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1395 self.assertTrue(bb.fetch.trusted_network(self.d, url))
1396
1397 def test_port_trusted_network(self):
1398 # Ensure trusted_network returns True, even if the url specifies a port.
1399 url = "git://someserver.org:8080/foo;rev=1;branch=master"
1400 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org")
1401 self.assertTrue(bb.fetch.trusted_network(self.d, url))
1402
1403 def test_untrusted_network(self):
1404 # Ensure trusted_network returns False when the host is NOT in the list.
1405 url = "git://someserver.org/foo;rev=1;branch=master"
1406 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
1407 self.assertFalse(bb.fetch.trusted_network(self.d, url))
1408
1409 def test_wild_untrusted_network(self):
1410 # Ensure trusted_network returns False when the host is NOT in the list.
1411 url = "git://*.someserver.org/foo;rev=1;branch=master"
1412 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
1413 self.assertFalse(bb.fetch.trusted_network(self.d, url))
1414
1415class URLHandle(unittest.TestCase):
1416 # Quote password as per RFC3986
1417 password = urllib.parse.quote(r"!#$%^&*()-_={}[]\|:?,.<>~`", r"!$&'/()*+,;=")
1418 datatable = {
1419 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}),
1420 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}),
1421 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])),
1422 "git://git.openembedded.org/bitbake;branch=@foo;protocol=https" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo', 'protocol' : 'https'}),
1423 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}),
1424 "file://example@.service": ('file', '', 'example@.service', '', '', {}),
1425 "https://somesite.com/somerepo.git;user=anyUser:idtoken=1234" : ('https', 'somesite.com', '/somerepo.git', '', '', {'user': 'anyUser:idtoken=1234'}),
1426 'git://s.o-me_ONE:%s@git.openembedded.org/bitbake;branch=main;protocol=https' % password: ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', password, {'branch': 'main', 'protocol' : 'https'}),
1427 }
1428 # we require a pathname to encodeurl but users can still pass such urls to
1429 # decodeurl and we need to handle them
1430 decodedata = datatable.copy()
1431 decodedata.update({
1432 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}),
1433 "npmsw://some.registry.url;package=@pkg;version=latest": ('npmsw', 'some.registry.url', '/', '', '', {'package': '@pkg', 'version': 'latest'}),
1434 })
1435
1436 def test_decodeurl(self):
1437 for k, v in self.decodedata.items():
1438 result = bb.fetch.decodeurl(k)
1439 self.assertEqual(result, v)
1440
1441 def test_encodeurl(self):
1442 for k, v in self.datatable.items():
1443 result = bb.fetch.encodeurl(v)
1444 if result.startswith("file:"):
1445 result = urllib.parse.unquote(result)
1446 self.assertEqual(result, k)
1447
1448class FetchLatestVersionTest(FetcherTest):
1449
1450 test_git_uris = {
1451 # version pattern "X.Y.Z"
1452 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "", "")
1453 : "1.99.4",
1454 # version pattern "vX.Y"
1455 # mirror of git.infradead.org since network issues interfered with testing
1456 ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master;protocol=https", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "", "")
1457 : "1.5.0",
1458 # version pattern "pkg_name-X.Y"
1459 # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing
1460 ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master;protocol=https", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "", "")
1461 : "1.0",
1462 # version pattern "pkg_name-vX.Y.Z"
1463 ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master;protocol=https", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "", "")
1464 : "1.4.0",
1465 # combination version pattern
1466 ("sysprof", "git://git.yoctoproject.org/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "", "")
1467 : "1.2.0",
1468 ("u-boot-mkimage", "git://git.yoctoproject.org/bbfetchtests-u-boot.git;branch=master;protocol=https", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "")
1469 : "2014.01",
1470 # version pattern "yyyymmdd"
1471 ("mobile-broadband-provider-info", "git://git.yoctoproject.org/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "", "")
1472 : "20120614",
1473 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
1474 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing
1475 ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master;protocol=https", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))", "")
1476 : "0.4.3",
1477 ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master;protocol=https", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))", "")
1478 : "11.0.0",
1479 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))", "")
1480 : "1.3.59",
1481 ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))", "")
1482 : "3.82+dbg0.9",
1483 ("sysdig", "git://github.com/draios/sysdig.git;branch=dev;protocol=https", "4fb6288275f567f63515df0ff0a6518043ecfa9b", r"^(?P<pver>\d+(\.\d+)+)", "10.0.0")
1484 : "0.28.0",
1485 }
1486
1487 WgetTestData = collections.namedtuple("WgetTestData", ["pn", "path", "pv", "check_uri", "check_regex"], defaults=[None, None, None])
1488 test_wget_uris = {
1489 #
1490 # packages with versions inside directory name
1491 #
1492 # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2
1493 WgetTestData("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2")
1494 : "2.24.2",
1495 # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz
1496 WgetTestData("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz")
1497 : "1.6.0",
1498 # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz
1499 WgetTestData("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz")
1500 : "2.8.12.1",
1501 # https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz
1502 WgetTestData("libxml2", "/software/libxml2/2.9/libxml2-2.9.14.tar.xz")
1503 : "2.10.3",
1504 #
1505 # packages with versions only in current directory
1506 #
1507 # https://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2
1508 WgetTestData("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2")
1509 : "2.19",
1510 # https://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2
1511 WgetTestData("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2")
1512 : "20120814",
1513 #
1514 # packages with "99" in the name of possible version
1515 #
1516 # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz
1517 WgetTestData("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz")
1518 : "5.0",
1519 # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2
1520 WgetTestData("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2")
1521 : "1.15.1",
1522 #
1523 # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX
1524 #
1525 # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2
1526 # https://github.com/apple/cups/releases
1527 WgetTestData("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", check_uri="/apple/cups/releases", check_regex=r"(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz")
1528 : "2.0.0",
1529 # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz
1530 # http://ftp.debian.org/debian/pool/main/d/db5.3/
1531 WgetTestData("db", "/berkeley-db/db-5.3.21.tar.gz", check_uri="/debian/pool/main/d/db5.3/", check_regex=r"(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz")
1532 : "5.3.10",
1533 #
1534 # packages where the tarball compression changed in the new version
1535 #
1536 # http://ftp.debian.org/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz
1537 WgetTestData("minicom", "/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz")
1538 : "2.8",
1539
1540 #
1541 # packages where the path doesn't actually contain the filename, so downloadfilename should be respected
1542 #
1543 WgetTestData("miniupnpd", "/software/miniupnp/download.php?file=miniupnpd_2.1.20191006.tar.gz;downloadfilename=miniupnpd_2.1.20191006.tar.gz", pv="2.1.20191006", check_uri="/software/miniupnp/download.php", check_regex=r"miniupnpd-(?P<pver>\d+(\.\d+)+)\.tar")
1544 : "2.3.7",
1545 }
1546
1547 test_crate_uris = {
1548 # basic example; version pattern "A.B.C+cargo-D.E.F"
1549 ("cargo-c", "crate://crates.io/cargo-c/0.9.18+cargo-0.69")
1550 : "0.9.29"
1551 }
1552
1553 @skipIfNoNetwork()
1554 def test_git_latest_versionstring(self):
1555 for k, v in self.test_git_uris.items():
1556 with self.subTest(pn=k[0]):
1557 self.d.setVar("PN", k[0])
1558 self.d.setVar("SRCREV", k[2])
1559 self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3])
1560 ud = bb.fetch2.FetchData(k[1], self.d)
1561 pupver= ud.method.latest_versionstring(ud, self.d)
1562 verstring = pupver[0]
1563 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
1564 r = bb.utils.vercmp_string(v, verstring)
1565 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
1566 if k[4]:
1567 r = bb.utils.vercmp_string(verstring, k[4])
1568 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], verstring, k[4]))
1569
1570 def test_wget_latest_versionstring(self):
1571 testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata"
1572 server = HTTPService(testdata, host="127.0.0.1")
1573 server.start()
1574 port = server.port
1575 try:
1576 for data, v in self.test_wget_uris.items():
1577 with self.subTest(pn=data.pn):
1578 self.d.setVar("PN", data.pn)
1579 self.d.setVar("PV", data.pv)
1580 if data.check_uri:
1581 checkuri = "http://127.0.0.1:%s/%s" % (port, data.check_uri)
1582 self.d.setVar("UPSTREAM_CHECK_URI", checkuri)
1583 if data.check_regex:
1584 self.d.setVar("UPSTREAM_CHECK_REGEX", data.check_regex)
1585
1586 url = "http://127.0.0.1:%s/%s" % (port, data.path)
1587 ud = bb.fetch2.FetchData(url, self.d)
1588 pupver = ud.method.latest_versionstring(ud, self.d)
1589 verstring = pupver[0]
1590 self.assertTrue(verstring, msg="Could not find upstream version for %s" % data.pn)
1591 r = bb.utils.vercmp_string(v, verstring)
1592 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (data.pn, v, verstring))
1593 finally:
1594 server.stop()
1595
1596 @skipIfNoNetwork()
1597 def test_crate_latest_versionstring(self):
1598 for k, v in self.test_crate_uris.items():
1599 with self.subTest(pn=k[0]):
1600 self.d.setVar("PN", k[0])
1601 ud = bb.fetch2.FetchData(k[1], self.d)
1602 pupver = ud.method.latest_versionstring(ud, self.d)
1603 verstring = pupver[0]
1604 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
1605 r = bb.utils.vercmp_string(v, verstring)
1606 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
1607
1608class FetchCheckStatusTest(FetcherTest):
1609 test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
1610 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
1611 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
1612 "https://yoctoproject.org/",
1613 "https://docs.yoctoproject.org",
1614 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
1615 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
1616 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz",
1617 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD
1618 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz"
1619 ]
1620
1621 @skipIfNoNetwork()
1622 def test_wget_checkstatus(self):
1623 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d)
1624 for u in self.test_wget_uris:
1625 with self.subTest(url=u):
1626 ud = fetch.ud[u]
1627 m = ud.method
1628 ret = m.checkstatus(fetch, ud, self.d)
1629 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
1630
1631 @skipIfNoNetwork()
1632 def test_wget_checkstatus_connection_cache(self):
1633 from bb.fetch2 import FetchConnectionCache
1634
1635 connection_cache = FetchConnectionCache()
1636 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d,
1637 connection_cache = connection_cache)
1638
1639 for u in self.test_wget_uris:
1640 with self.subTest(url=u):
1641 ud = fetch.ud[u]
1642 m = ud.method
1643 ret = m.checkstatus(fetch, ud, self.d)
1644 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
1645
1646 connection_cache.close_connections()
1647
1648
1649class GitMakeShallowTest(FetcherTest):
1650 def setUp(self):
1651 FetcherTest.setUp(self)
1652 self.gitdir = os.path.join(self.tempdir, 'gitshallow')
1653 bb.utils.mkdirhier(self.gitdir)
1654 self.git_init()
1655
1656 def assertRefs(self, expected_refs):
1657 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
1658 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines()
1659 self.assertEqual(sorted(full_expected), sorted(actual_refs))
1660
1661 def assertRevCount(self, expected_count, args=None):
1662 if args is None:
1663 args = ['HEAD']
1664 revs = self.git(['rev-list'] + args)
1665 actual_count = len(revs.splitlines())
1666 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1667
1668 def make_shallow(self, args=None):
1669 if args is None:
1670 args = ['HEAD']
1671 return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir)
1672
1673 def add_empty_file(self, path, msg=None):
1674 if msg is None:
1675 msg = path
1676 open(os.path.join(self.gitdir, path), 'w').close()
1677 self.git(['add', path])
1678 self.git(['commit', '-m', msg, path])
1679
1680 def test_make_shallow_single_branch_no_merge(self):
1681 self.add_empty_file('a')
1682 self.add_empty_file('b')
1683 self.assertRevCount(2)
1684 self.make_shallow()
1685 self.assertRevCount(1)
1686
1687 def test_make_shallow_single_branch_one_merge(self):
1688 self.add_empty_file('a')
1689 self.add_empty_file('b')
1690 self.git('checkout -b a_branch')
1691 self.add_empty_file('c')
1692 self.git('checkout master')
1693 self.add_empty_file('d')
1694 self.git('merge --no-ff --no-edit a_branch')
1695 self.git('branch -d a_branch')
1696 self.add_empty_file('e')
1697 self.assertRevCount(6)
1698 self.make_shallow(['HEAD~2'])
1699 self.assertRevCount(5)
1700
1701 def test_make_shallow_at_merge(self):
1702 self.add_empty_file('a')
1703 self.git('checkout -b a_branch')
1704 self.add_empty_file('b')
1705 self.git('checkout master')
1706 self.git('merge --no-ff --no-edit a_branch')
1707 self.git('branch -d a_branch')
1708 self.assertRevCount(3)
1709 self.make_shallow()
1710 self.assertRevCount(1)
1711
1712 def test_make_shallow_annotated_tag(self):
1713 self.add_empty_file('a')
1714 self.add_empty_file('b')
1715 self.git('tag -a -m a_tag a_tag')
1716 self.assertRevCount(2)
1717 self.make_shallow(['a_tag'])
1718 self.assertRevCount(1)
1719
1720 def test_make_shallow_multi_ref(self):
1721 self.add_empty_file('a')
1722 self.add_empty_file('b')
1723 self.git('checkout -b a_branch')
1724 self.add_empty_file('c')
1725 self.git('checkout master')
1726 self.add_empty_file('d')
1727 self.git('checkout -b a_branch_2')
1728 self.add_empty_file('a_tag')
1729 self.git('tag a_tag')
1730 self.git('checkout master')
1731 self.git('branch -D a_branch_2')
1732 self.add_empty_file('e')
1733 self.assertRevCount(6, ['--all'])
1734 self.make_shallow()
1735 self.assertRevCount(5, ['--all'])
1736
1737 def test_make_shallow_multi_ref_trim(self):
1738 self.add_empty_file('a')
1739 self.git('checkout -b a_branch')
1740 self.add_empty_file('c')
1741 self.git('checkout master')
1742 self.assertRevCount(1)
1743 self.assertRevCount(2, ['--all'])
1744 self.assertRefs(['master', 'a_branch'])
1745 self.make_shallow(['-r', 'master', 'HEAD'])
1746 self.assertRevCount(1, ['--all'])
1747 self.assertRefs(['master'])
1748
1749 def test_make_shallow_noop(self):
1750 self.add_empty_file('a')
1751 self.assertRevCount(1)
1752 self.make_shallow()
1753 self.assertRevCount(1)
1754
1755 @skipIfNoNetwork()
1756 def test_make_shallow_bitbake(self):
1757 self.git('remote add origin https://github.com/openembedded/bitbake')
1758 self.git('fetch --tags origin')
1759 orig_revs = len(self.git('rev-list --all').splitlines())
1760 self.make_shallow(['refs/tags/1.10.0'])
1761 self.assertRevCount(orig_revs - 1746, ['--all'])
1762
1763class GitShallowTest(FetcherTest):
1764 def setUp(self):
1765 FetcherTest.setUp(self)
1766 self.gitdir = os.path.join(self.tempdir, 'git')
1767 self.srcdir = os.path.join(self.tempdir, 'gitsource')
1768
1769 bb.utils.mkdirhier(self.srcdir)
1770 self.git_init(cwd=self.srcdir)
1771 self.d.setVar('WORKDIR', self.tempdir)
1772 self.d.setVar('S', self.gitdir)
1773 self.d.delVar('PREMIRRORS')
1774 self.d.delVar('MIRRORS')
1775
1776 uri = 'git://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
1777 self.d.setVar('SRC_URI', uri)
1778 self.d.setVar('SRCREV', '${AUTOREV}')
1779 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
1780
1781 self.d.setVar('BB_GIT_SHALLOW', '1')
1782 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
1783 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
1784 self.d.setVar("__BBSRCREV_SEEN", "1")
1785
1786 def assertRefs(self, expected_refs, cwd=None):
1787 if cwd is None:
1788 cwd = self.gitdir
1789 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines()
1790 # Resolve references into the same format as the comparision (needed by git 2.48 onwards)
1791 actual_refs = self.git(['rev-parse', '--symbolic-full-name'] + actual_refs, cwd=cwd).splitlines()
1792 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines()
1793 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs)))
1794
1795 def assertRevCount(self, expected_count, args=None, cwd=None):
1796 if args is None:
1797 args = ['HEAD']
1798 if cwd is None:
1799 cwd = self.gitdir
1800 revs = self.git(['rev-list'] + args, cwd=cwd)
1801 actual_count = len(revs.splitlines())
1802 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1803
1804 def add_empty_file(self, path, cwd=None, msg=None):
1805 if msg is None:
1806 msg = path
1807 if cwd is None:
1808 cwd = self.srcdir
1809 open(os.path.join(cwd, path), 'w').close()
1810 self.git(['add', path], cwd)
1811 self.git(['commit', '-m', msg, path], cwd)
1812
1813 def fetch(self, uri=None):
1814 if uri is None:
1815 uris = self.d.getVar('SRC_URI').split()
1816 uri = uris[0]
1817 d = self.d
1818 else:
1819 d = self.d.createCopy()
1820 d.setVar('SRC_URI', uri)
1821 uri = d.expand(uri)
1822 uris = [uri]
1823
1824 fetcher = bb.fetch2.Fetch(uris, d)
1825 fetcher.download()
1826 ud = fetcher.ud[uri]
1827 return fetcher, ud
1828
1829 def fetch_and_unpack(self, uri=None):
1830 fetcher, ud = self.fetch(uri)
1831 fetcher.unpack(self.d.getVar('WORKDIR'))
1832 assert os.path.exists(self.d.getVar('S'))
1833 return fetcher, ud
1834
1835 def fetch_shallow(self, uri=None, disabled=False, keepclone=False):
1836 """Fetch a uri, generating a shallow tarball, then unpack using it"""
1837 fetcher, ud = self.fetch_and_unpack(uri)
1838
1839 # Confirm that the unpacked repo is unshallow
1840 if not disabled:
1841 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1842
1843 # fetch and unpack, from the shallow tarball
1844 bb.utils.remove(self.gitdir, recurse=True)
1845 if os.path.exists(ud.clonedir):
1846 bb.process.run('chmod u+w -R "%s"' % ud.clonedir)
1847 bb.utils.remove(ud.clonedir, recurse=True)
1848 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
1849
1850 # confirm that the unpacked repo is used when no git clone or git
1851 # mirror tarball is available
1852 fetcher, ud = self.fetch_and_unpack(uri)
1853 if not disabled:
1854 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir
1855 else:
1856 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir
1857 return fetcher, ud
1858
1859 def test_shallow_disabled(self):
1860 self.add_empty_file('a')
1861 self.add_empty_file('b')
1862 self.assertRevCount(2, cwd=self.srcdir)
1863
1864 self.d.setVar('BB_GIT_SHALLOW', '0')
1865 self.fetch_shallow(disabled=True)
1866 self.assertRevCount(2)
1867
1868 def test_shallow_nobranch(self):
1869 self.add_empty_file('a')
1870 self.add_empty_file('b')
1871 self.assertRevCount(2, cwd=self.srcdir)
1872
1873 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip()
1874 self.d.setVar('SRCREV', srcrev)
1875 uri = self.d.getVar('SRC_URI').split()[0]
1876 uri = '%s;nobranch=1;bare=1' % uri
1877
1878 self.fetch_shallow(uri)
1879 self.assertRevCount(1)
1880
1881 # shallow refs are used to ensure the srcrev sticks around when we
1882 # have no other branches referencing it
1883 self.assertRefs(['refs/shallow/default'])
1884
1885 def test_shallow_default_depth_1(self):
1886 # Create initial git repo
1887 self.add_empty_file('a')
1888 self.add_empty_file('b')
1889 self.assertRevCount(2, cwd=self.srcdir)
1890
1891 self.fetch_shallow()
1892 self.assertRevCount(1)
1893
1894 def test_shallow_depth_0_disables(self):
1895 self.add_empty_file('a')
1896 self.add_empty_file('b')
1897 self.assertRevCount(2, cwd=self.srcdir)
1898
1899 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1900 self.fetch_shallow(disabled=True)
1901 self.assertRevCount(2)
1902
1903 def test_shallow_depth_default_override(self):
1904 self.add_empty_file('a')
1905 self.add_empty_file('b')
1906 self.assertRevCount(2, cwd=self.srcdir)
1907
1908 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2')
1909 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1')
1910 self.fetch_shallow()
1911 self.assertRevCount(1)
1912
1913 def test_shallow_depth_default_override_disable(self):
1914 self.add_empty_file('a')
1915 self.add_empty_file('b')
1916 self.add_empty_file('c')
1917 self.assertRevCount(3, cwd=self.srcdir)
1918
1919 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1920 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2')
1921 self.fetch_shallow()
1922 self.assertRevCount(2)
1923
1924 def test_current_shallow_out_of_date_clone(self):
1925 # Create initial git repo
1926 self.add_empty_file('a')
1927 self.add_empty_file('b')
1928 self.add_empty_file('c')
1929 self.assertRevCount(3, cwd=self.srcdir)
1930
1931 # Clone without tarball
1932 self.d.setVar('BB_GIT_SHALLOW', '0')
1933 fetcher, ud = self.fetch()
1934
1935 # Clone and generate mirror tarball
1936 self.d.setVar('BB_GIT_SHALLOW', '1')
1937 fetcher, ud = self.fetch()
1938
1939 # Ensure we have a current mirror tarball, but an out of date clone
1940 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir)
1941 self.assertRevCount(2, cwd=ud.clonedir)
1942
1943 # Fetch and unpack, from the current tarball, not the out of date clone
1944 bb.utils.remove(self.gitdir, recurse=True)
1945 fetcher, ud = self.fetch()
1946 fetcher.unpack(self.d.getVar('WORKDIR'))
1947 self.assertRevCount(1)
1948 assert os.path.exists(os.path.join(self.d.getVar('WORKDIR'), 'git', 'c'))
1949
1950 def test_shallow_single_branch_no_merge(self):
1951 self.add_empty_file('a')
1952 self.add_empty_file('b')
1953 self.assertRevCount(2, cwd=self.srcdir)
1954
1955 self.fetch_shallow()
1956 self.assertRevCount(1)
1957 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1958 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1959
1960 def test_shallow_no_dangling(self):
1961 self.add_empty_file('a')
1962 self.add_empty_file('b')
1963 self.assertRevCount(2, cwd=self.srcdir)
1964
1965 self.fetch_shallow()
1966 self.assertRevCount(1)
1967 assert not self.git('fsck --dangling')
1968
1969 def test_shallow_srcrev_branch_truncation(self):
1970 self.add_empty_file('a')
1971 self.add_empty_file('b')
1972 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip()
1973 self.add_empty_file('c')
1974 self.assertRevCount(3, cwd=self.srcdir)
1975
1976 self.d.setVar('SRCREV', b_commit)
1977 self.fetch_shallow()
1978
1979 # The 'c' commit was removed entirely, and 'a' was removed from history
1980 self.assertRevCount(1, ['--all'])
1981 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit)
1982 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1983 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1984 assert not os.path.exists(os.path.join(self.gitdir, 'c'))
1985
1986 def test_shallow_ref_pruning(self):
1987 self.add_empty_file('a')
1988 self.add_empty_file('b')
1989 self.git('branch a_branch', cwd=self.srcdir)
1990 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
1991 self.assertRevCount(2, cwd=self.srcdir)
1992
1993 self.fetch_shallow()
1994
1995 self.assertRefs(['master', 'origin/master'])
1996 self.assertRevCount(1)
1997
1998 def test_shallow_submodules(self):
1999 self.add_empty_file('a')
2000 self.add_empty_file('b')
2001
2002 smdir = os.path.join(self.tempdir, 'gitsubmodule')
2003 bb.utils.mkdirhier(smdir)
2004 self.git_init(cwd=smdir)
2005 # Make this look like it was cloned from a remote...
2006 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
2007 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
2008 self.add_empty_file('asub', cwd=smdir)
2009 self.add_empty_file('bsub', cwd=smdir)
2010
2011 self.git('submodule init', cwd=self.srcdir)
2012 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir)
2013 self.git('submodule update', cwd=self.srcdir)
2014 self.git('commit -m submodule -a', cwd=self.srcdir)
2015
2016 uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
2017 fetcher, ud = self.fetch_shallow(uri)
2018
2019 # Verify the main repository is shallow
2020 self.assertRevCount(1)
2021
2022 # Verify the gitsubmodule directory is present
2023 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
2024
2025 # Verify the submodule is also shallow
2026 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
2027
2028 def test_shallow_submodule_mirrors(self):
2029 self.add_empty_file('a')
2030 self.add_empty_file('b')
2031
2032 smdir = os.path.join(self.tempdir, 'gitsubmodule')
2033 bb.utils.mkdirhier(smdir)
2034 self.git_init(cwd=smdir)
2035 # Make this look like it was cloned from a remote...
2036 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
2037 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
2038 self.add_empty_file('asub', cwd=smdir)
2039 self.add_empty_file('bsub', cwd=smdir)
2040
2041 self.git('submodule init', cwd=self.srcdir)
2042 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir)
2043 self.git('submodule update', cwd=self.srcdir)
2044 self.git('commit -m submodule -a', cwd=self.srcdir)
2045
2046 uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
2047
2048 # Fetch once to generate the shallow tarball
2049 fetcher, ud = self.fetch(uri)
2050
2051 # Set up the mirror
2052 mirrordir = os.path.join(self.tempdir, 'mirror')
2053 bb.utils.rename(self.dldir, mirrordir)
2054 self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/' % mirrordir)
2055
2056 # Fetch from the mirror
2057 bb.utils.remove(self.dldir, recurse=True)
2058 bb.utils.remove(self.gitdir, recurse=True)
2059 self.fetch_and_unpack(uri)
2060
2061 # Verify the main repository is shallow
2062 self.assertRevCount(1)
2063
2064 # Verify the gitsubmodule directory is present
2065 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
2066
2067 # Verify the submodule is also shallow
2068 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
2069
2070 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')):
2071 def test_shallow_annex(self):
2072 self.add_empty_file('a')
2073 self.add_empty_file('b')
2074 self.git('annex init', cwd=self.srcdir)
2075 open(os.path.join(self.srcdir, 'c'), 'w').close()
2076 self.git('annex add c', cwd=self.srcdir)
2077 self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir)
2078 bb.process.run('chmod u+w -R %s' % self.srcdir)
2079
2080 uri = 'gitannex://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
2081 fetcher, ud = self.fetch_shallow(uri)
2082
2083 self.assertRevCount(1)
2084 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0]
2085 assert os.path.exists(os.path.join(self.gitdir, 'c'))
2086
2087 def test_shallow_clone_preferred_over_shallow(self):
2088 self.add_empty_file('a')
2089 self.add_empty_file('b')
2090
2091 # Fetch once to generate the shallow tarball
2092 self.d.setVar('BB_GIT_SHALLOW', '0')
2093 fetcher, ud = self.fetch()
2094
2095 # Fetch and unpack with both the clonedir and shallow tarball available
2096 bb.utils.remove(self.gitdir, recurse=True)
2097 self.d.setVar('BB_GIT_SHALLOW', '1')
2098 fetcher, ud = self.fetch_and_unpack()
2099
2100 # The unpacked tree should *not* be shallow
2101 self.assertRevCount(2)
2102 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow'))
2103
2104 def test_shallow_mirrors(self):
2105 self.add_empty_file('a')
2106 self.add_empty_file('b')
2107
2108 # Fetch once to generate the shallow tarball
2109 fetcher, ud = self.fetch()
2110 mirrortarball = ud.mirrortarballs[0]
2111 assert os.path.exists(os.path.join(self.dldir, mirrortarball))
2112
2113 # Set up the mirror
2114 mirrordir = os.path.join(self.tempdir, 'mirror')
2115 bb.utils.mkdirhier(mirrordir)
2116 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % mirrordir)
2117
2118 bb.utils.rename(os.path.join(self.dldir, mirrortarball),
2119 os.path.join(mirrordir, mirrortarball))
2120
2121 # Fetch from the mirror
2122 bb.utils.remove(self.dldir, recurse=True)
2123 bb.utils.remove(self.gitdir, recurse=True)
2124 self.fetch_and_unpack()
2125 self.assertRevCount(1)
2126
2127 def test_shallow_invalid_depth(self):
2128 self.add_empty_file('a')
2129 self.add_empty_file('b')
2130
2131 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12')
2132 with self.assertRaises(bb.fetch2.FetchError):
2133 self.fetch()
2134
2135 def test_shallow_invalid_depth_default(self):
2136 self.add_empty_file('a')
2137 self.add_empty_file('b')
2138
2139 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12')
2140 with self.assertRaises(bb.fetch2.FetchError):
2141 self.fetch()
2142
2143 def test_shallow_extra_refs(self):
2144 self.add_empty_file('a')
2145 self.add_empty_file('b')
2146 self.git('branch a_branch', cwd=self.srcdir)
2147 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
2148 self.assertRevCount(2, cwd=self.srcdir)
2149
2150 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch')
2151 self.fetch_shallow()
2152
2153 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
2154 self.assertRevCount(1)
2155
2156 def test_shallow_extra_refs_wildcard(self):
2157 self.add_empty_file('a')
2158 self.add_empty_file('b')
2159 self.git('branch a_branch', cwd=self.srcdir)
2160 self.git('tag v1.0', cwd=self.srcdir)
2161 self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir)
2162 self.assertRevCount(2, cwd=self.srcdir)
2163
2164 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
2165 self.fetch_shallow()
2166
2167 self.assertRefs(['master', 'origin/master', 'v1.0'])
2168 self.assertRevCount(1)
2169
2170 def test_shallow_missing_extra_refs(self):
2171 self.add_empty_file('a')
2172 self.add_empty_file('b')
2173
2174 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo')
2175 with self.assertRaises(bb.fetch2.FetchError):
2176 self.fetch()
2177
2178 def test_shallow_missing_extra_refs_wildcard(self):
2179 self.add_empty_file('a')
2180 self.add_empty_file('b')
2181
2182 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
2183 self.fetch()
2184
2185 def test_shallow_remove_revs(self):
2186 # Create initial git repo
2187 self.add_empty_file('a')
2188 self.add_empty_file('b')
2189 self.git('checkout -b a_branch', cwd=self.srcdir)
2190 self.add_empty_file('c')
2191 self.add_empty_file('d')
2192 self.git('checkout master', cwd=self.srcdir)
2193 self.git('tag v0.0 a_branch', cwd=self.srcdir)
2194 self.add_empty_file('e')
2195 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
2196 self.git('branch -d a_branch', cwd=self.srcdir)
2197 self.add_empty_file('f')
2198 self.assertRevCount(7, cwd=self.srcdir)
2199
2200 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2201 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
2202
2203 self.fetch_shallow()
2204
2205 self.assertRevCount(2)
2206
2207 def test_shallow_invalid_revs(self):
2208 self.add_empty_file('a')
2209 self.add_empty_file('b')
2210
2211 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2212 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
2213
2214 with self.assertRaises(bb.fetch2.FetchError):
2215 self.fetch()
2216
2217 def test_shallow_fetch_missing_revs(self):
2218 self.add_empty_file('a')
2219 self.add_empty_file('b')
2220 fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
2221 self.git('tag v0.0 master', cwd=self.srcdir)
2222 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2223 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
2224
2225 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm:
2226 self.fetch_shallow()
2227 self.assertIn("fatal: no commits selected for shallow requests", cm.output[0])
2228
2229 def test_shallow_fetch_missing_revs_fails(self):
2230 self.add_empty_file('a')
2231 self.add_empty_file('b')
2232 fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
2233 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2234 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
2235
2236 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm:
2237 self.fetch_shallow()
2238 self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0])
2239
2240 @skipIfNoNetwork()
2241 def test_git_shallow_fetch_premirrors(self):
2242 url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
2243
2244 # Create a separate premirror directory within tempdir
2245 premirror = os.path.join(self.tempdir, "premirror")
2246 os.mkdir(premirror)
2247
2248 # Fetch a non-shallow clone into the premirror subdir
2249 self.d.setVar('BB_GIT_SHALLOW', '0')
2250 self.d.setVar("DL_DIR", premirror)
2251 fetcher, ud = self.fetch(url)
2252
2253 # Fetch a shallow clone from the premirror subdir with unpacking
2254 # using the original recipe URL and the premirror mapping
2255 self.d.setVar('BB_GIT_SHALLOW', '1')
2256 self.d.setVar("DL_DIR", self.dldir)
2257 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
2258 self.d.setVar('BB_NO_NETWORK', '1')
2259 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
2260 self.d.setVar("PREMIRRORS", "git://.*/.* git://{0};protocol=file".format(premirror + "/git2/" + ud.host + ud.path.replace("/", ".")))
2261 fetcher = self.fetch_and_unpack(url)
2262
2263 # Verify that the unpacked sources are shallow clones
2264 self.assertRevCount(1)
2265 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow'))
2266
2267 @skipIfNoNetwork()
2268 def test_bitbake(self):
2269 self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir)
2270 self.git('config core.bare true', cwd=self.srcdir)
2271 self.git('fetch', cwd=self.srcdir)
2272
2273 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2274 # Note that the 1.10.0 tag is annotated, so this also tests
2275 # reference of an annotated vs unannotated tag
2276 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0')
2277
2278 self.fetch_shallow()
2279
2280 # Confirm that the history of 1.10.0 was removed
2281 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines())
2282 revs = len(self.git('rev-list master').splitlines())
2283 self.assertNotEqual(orig_revs, revs)
2284 self.assertRefs(['master', 'origin/master'])
2285 self.assertRevCount(orig_revs - 1760)
2286
2287 def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self):
2288 self.add_empty_file('a')
2289 fetcher, ud = self.fetch()
2290 bb.utils.remove(self.gitdir, recurse=True)
2291 bb.utils.remove(self.dldir, recurse=True)
2292
2293 with self.assertRaises(bb.fetch2.UnpackError) as context:
2294 fetcher.unpack(self.d.getVar('WORKDIR'))
2295
2296 self.assertIn("No up to date source found", context.exception.msg)
2297 self.assertIn("clone directory not available or not up to date", context.exception.msg)
2298
2299 def test_shallow_check_is_shallow(self):
2300 self.add_empty_file('a')
2301 self.add_empty_file('b')
2302
2303 # Fetch and unpack without the clonedir and *only* shallow tarball available
2304 bb.utils.remove(self.gitdir, recurse=True)
2305 fetcher, ud = self.fetch_and_unpack()
2306
2307 # The unpacked tree *should* be shallow
2308 self.assertRevCount(1)
2309 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow'))
2310
2311 def test_shallow_succeeds_with_tag_containing_slash(self):
2312 self.add_empty_file('a')
2313 self.add_empty_file('b')
2314 self.git('tag t1/t2/t3', cwd=self.srcdir)
2315 self.assertRevCount(2, cwd=self.srcdir)
2316
2317 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip()
2318 self.d.setVar('SRCREV', srcrev)
2319 uri = self.d.getVar('SRC_URI').split()[0]
2320 uri = '%s;tag=t1/t2/t3' % uri
2321 self.fetch_shallow(uri)
2322 self.assertRevCount(1)
2323
2324class GitLfsTest(FetcherTest):
2325 def skipIfNoGitLFS():
2326 if not shutil.which('git-lfs'):
2327 return unittest.skip('git-lfs not installed')
2328 return lambda f: f
2329
2330 def setUp(self):
2331 FetcherTest.setUp(self)
2332
2333 self.gitdir = os.path.join(self.tempdir, 'git')
2334 self.srcdir = os.path.join(self.tempdir, 'gitsource')
2335
2336 self.d.setVar('WORKDIR', self.tempdir)
2337 self.d.setVar('S', self.gitdir)
2338 self.d.delVar('PREMIRRORS')
2339 self.d.delVar('MIRRORS')
2340
2341 self.d.setVar('SRCREV', '${AUTOREV}')
2342 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
2343 self.d.setVar("__BBSRCREV_SEEN", "1")
2344
2345 bb.utils.mkdirhier(self.srcdir)
2346 self.git_init(cwd=self.srcdir)
2347 self.commit_file('.gitattributes', '*.mp3 filter=lfs -text')
2348
2349 def commit(self, *, cwd=None):
2350 cwd = cwd or self.srcdir
2351 self.git(["commit", "-m", "Change"], cwd=cwd)
2352 return self.git(["rev-parse", "HEAD"], cwd=cwd).strip()
2353
2354 def commit_file(self, filename, content, *, cwd=None):
2355 cwd = cwd or self.srcdir
2356
2357 with open(os.path.join(cwd, filename), "w") as f:
2358 f.write(content)
2359 self.git(["add", filename], cwd=cwd)
2360 return self.commit(cwd=cwd)
2361
2362 def fetch(self, uri=None, download=True):
2363 uris = self.d.getVar('SRC_URI').split()
2364 uri = uris[0]
2365 d = self.d
2366
2367 fetcher = bb.fetch2.Fetch(uris, d)
2368 if download:
2369 fetcher.download()
2370 ud = fetcher.ud[uri]
2371 return fetcher, ud
2372
2373 def get_real_git_lfs_file(self):
2374 self.d.setVar('PATH', os.environ.get('PATH'))
2375 fetcher, ud = self.fetch()
2376 fetcher.unpack(self.d.getVar('WORKDIR'))
2377 unpacked_lfs_file = os.path.join(self.d.getVar('WORKDIR'), 'git', "Cat_poster_1.jpg")
2378 return unpacked_lfs_file
2379
2380 @skipIfNoGitLFS()
2381 def test_gitsm_lfs(self):
2382 """Test that the gitsm fetcher caches objects stored via LFS"""
2383 self.git(["lfs", "install", "--local"], cwd=self.srcdir)
2384
2385 def fetch_and_verify(revision, filename, content):
2386 self.d.setVar('SRCREV', revision)
2387 fetcher, ud = self.fetch()
2388
2389 with hide_directory(submoduledir), hide_directory(self.srcdir):
2390 workdir = self.d.getVar('WORKDIR')
2391 fetcher.unpack(workdir)
2392
2393 with open(os.path.join(workdir, "git", filename)) as f:
2394 self.assertEqual(f.read(), content)
2395
2396 # Create the git repository that will later be used as a submodule
2397 submoduledir = self.tempdir + "/submodule"
2398 bb.utils.mkdirhier(submoduledir)
2399 self.git_init(submoduledir)
2400 self.git(["lfs", "install", "--local"], cwd=submoduledir)
2401 self.commit_file('.gitattributes', '*.mp3 filter=lfs -text', cwd=submoduledir)
2402
2403 submodule_commit_1 = self.commit_file("a.mp3", "submodule version 1", cwd=submoduledir)
2404 _ = self.commit_file("a.mp3", "submodule version 2", cwd=submoduledir)
2405
2406 # Add the submodule to the repository at its current HEAD revision
2407 self.git(["-c", "protocol.file.allow=always", "submodule", "add", submoduledir, "submodule"],
2408 cwd=self.srcdir)
2409 base_commit_1 = self.commit()
2410
2411 # Let the submodule point at a different revision
2412 self.git(["checkout", submodule_commit_1], self.srcdir + "/submodule")
2413 self.git(["add", "submodule"], cwd=self.srcdir)
2414 base_commit_2 = self.commit()
2415
2416 # Add a LFS file to the repository
2417 base_commit_3 = self.commit_file("a.mp3", "version 1")
2418 # Update the added LFS file
2419 base_commit_4 = self.commit_file("a.mp3", "version 2")
2420
2421 self.d.setVar('SRC_URI', "gitsm://%s;protocol=file;lfs=1;branch=master" % self.srcdir)
2422
2423 # Verify that LFS objects referenced from submodules are fetched and checked out
2424 fetch_and_verify(base_commit_1, "submodule/a.mp3", "submodule version 2")
2425 # Verify that the repository inside the download cache of a submodile is extended with any
2426 # additional LFS objects needed when checking out a different revision.
2427 fetch_and_verify(base_commit_2, "submodule/a.mp3", "submodule version 1")
2428 # Verify that LFS objects referenced from the base repository are fetched and checked out
2429 fetch_and_verify(base_commit_3, "a.mp3", "version 1")
2430 # Verify that the cached repository is extended with any additional LFS objects required
2431 # when checking out a different revision.
2432 fetch_and_verify(base_commit_4, "a.mp3", "version 2")
2433
2434 @skipIfNoGitLFS()
2435 def test_gitsm_lfs_disabled(self):
2436 """Test that the gitsm fetcher does not use LFS when explicitly disabled"""
2437 self.git(["lfs", "install", "--local"], cwd=self.srcdir)
2438
2439 def fetch_and_verify(revision, filename, content):
2440 self.d.setVar('SRCREV', revision)
2441 fetcher, ud = self.fetch()
2442
2443 with hide_directory(submoduledir), hide_directory(self.srcdir):
2444 workdir = self.d.getVar('WORKDIR')
2445 fetcher.unpack(workdir)
2446
2447 with open(os.path.join(workdir, "git", filename)) as f:
2448 # Assume that LFS did not perform smudging when the expected content is
2449 # missing.
2450 self.assertNotEqual(f.read(), content)
2451
2452 # Create the git repository that will later be used as a submodule
2453 submoduledir = self.tempdir + "/submodule"
2454 bb.utils.mkdirhier(submoduledir)
2455 self.git_init(submoduledir)
2456 self.git(["lfs", "install", "--local"], cwd=submoduledir)
2457 self.commit_file('.gitattributes', '*.mp3 filter=lfs -text', cwd=submoduledir)
2458
2459 submodule_commit_1 = self.commit_file("a.mp3", "submodule version 1", cwd=submoduledir)
2460
2461 # Add the submodule to the repository at its current HEAD revision
2462 self.git(["-c", "protocol.file.allow=always", "submodule", "add", submoduledir, "submodule"],
2463 cwd=self.srcdir)
2464 base_commit_1 = self.commit()
2465
2466 # Add a LFS file to the repository
2467 base_commit_2 = self.commit_file("a.mp3", "version 1")
2468
2469 self.d.setVar('SRC_URI', "gitsm://%s;protocol=file;lfs=1;branch=master;lfs=0" % self.srcdir)
2470
2471 # Verify that LFS objects referenced from submodules are not fetched nor checked out
2472 fetch_and_verify(base_commit_1, "submodule/a.mp3", "submodule version 1")
2473 # Verify that the LFS objects referenced from the base repository are not fetched nor
2474 # checked out
2475 fetch_and_verify(base_commit_2, "a.mp3", "version 1")
2476
2477 @skipIfNoGitLFS()
2478 def test_fetch_lfs_on_srcrev_change(self):
2479 """Test if fetch downloads missing LFS objects when a different revision within an existing repository is requested"""
2480 self.git(["lfs", "install", "--local"], cwd=self.srcdir)
2481
2482 def fetch_and_verify(revision, filename, content):
2483 self.d.setVar('SRCREV', revision)
2484 fetcher, ud = self.fetch()
2485
2486 with hide_directory(self.srcdir):
2487 workdir = self.d.getVar('WORKDIR')
2488 fetcher.unpack(workdir)
2489
2490 with open(os.path.join(workdir, "git", filename)) as f:
2491 self.assertEqual(f.read(), content)
2492
2493 commit_1 = self.commit_file("a.mp3", "version 1")
2494 commit_2 = self.commit_file("a.mp3", "version 2")
2495
2496 self.d.setVar('SRC_URI', "git://%s;protocol=file;lfs=1;branch=master" % self.srcdir)
2497
2498 # Seed the local download folder by fetching the latest commit and verifying that the LFS contents are
2499 # available even when the upstream repository disappears.
2500 fetch_and_verify(commit_2, "a.mp3", "version 2")
2501 # Verify that even when an older revision is fetched, the needed LFS objects are fetched into the download
2502 # folder.
2503 fetch_and_verify(commit_1, "a.mp3", "version 1")
2504
2505 @skipIfNoGitLFS()
2506 @skipIfNoNetwork()
2507 def test_real_git_lfs_repo_succeeds_without_lfs_param(self):
2508 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master")
2509 f = self.get_real_git_lfs_file()
2510 self.assertTrue(os.path.exists(f))
2511 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f))
2512
2513 @skipIfNoGitLFS()
2514 @skipIfNoNetwork()
2515 def test_real_git_lfs_repo_succeeds(self):
2516 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=1")
2517 f = self.get_real_git_lfs_file()
2518 self.assertTrue(os.path.exists(f))
2519 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f))
2520
2521 @skipIfNoGitLFS()
2522 @skipIfNoNetwork()
2523 def test_real_git_lfs_repo_skips(self):
2524 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=0")
2525 f = self.get_real_git_lfs_file()
2526 # This is the actual non-smudged placeholder file on the repo if git-lfs does not run
2527 lfs_file = (
2528 'version https://git-lfs.github.com/spec/v1\n'
2529 'oid sha256:34be66b1a39a1955b46a12588df9d5f6fc1da790e05cf01f3c7422f4bbbdc26b\n'
2530 'size 11423554\n'
2531 )
2532
2533 with open(f) as fh:
2534 self.assertEqual(lfs_file, fh.read())
2535
2536 @skipIfNoGitLFS()
2537 def test_lfs_enabled(self):
2538 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir
2539 self.d.setVar('SRC_URI', uri)
2540
2541 # With git-lfs installed, test that we can fetch and unpack
2542 fetcher, ud = self.fetch()
2543 shutil.rmtree(self.gitdir, ignore_errors=True)
2544 fetcher.unpack(self.d.getVar('WORKDIR'))
2545
2546 @skipIfNoGitLFS()
2547 def test_lfs_disabled(self):
2548 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir
2549 self.d.setVar('SRC_URI', uri)
2550
2551 # Verify that the fetcher can survive even if the source
2552 # repository has Git LFS usage configured.
2553 fetcher, ud = self.fetch()
2554 fetcher.unpack(self.d.getVar('WORKDIR'))
2555
2556 @skipIfNoGitLFS()
2557 def test_lfs_enabled_not_installed_during_unpack(self):
2558 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir
2559 self.d.setVar('SRC_URI', uri)
2560
2561 # Careful: suppress initial attempt at downloading
2562 fetcher, ud = self.fetch(uri=None, download=False)
2563
2564 fetcher.download()
2565 # If git-lfs cannot be found, the unpack should throw an error
2566 with self.assertRaises(bb.fetch2.FetchError):
2567 with unittest.mock.patch("shutil.which", return_value=None):
2568 shutil.rmtree(self.gitdir, ignore_errors=True)
2569 fetcher.unpack(self.d.getVar('WORKDIR'))
2570
2571 def test_lfs_enabled_not_installed(self):
2572 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir
2573 self.d.setVar('SRC_URI', uri)
2574
2575 # Careful: suppress initial attempt at downloading
2576 fetcher, ud = self.fetch(uri=None, download=False)
2577
2578 # If git-lfs cannot be found, the download should throw an error
2579 with unittest.mock.patch("shutil.which", return_value=None):
2580 with self.assertRaises(bb.fetch2.FetchError):
2581 fetcher.download()
2582
2583 def test_lfs_disabled_not_installed(self):
2584 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir
2585 self.d.setVar('SRC_URI', uri)
2586
2587 # Careful: suppress initial attempt at downloading
2588 fetcher, ud = self.fetch(uri=None, download=False)
2589
2590 # Even if git-lfs cannot be found, the download / unpack should be successful
2591 with unittest.mock.patch("shutil.which", return_value=None):
2592 fetcher.download()
2593 shutil.rmtree(self.gitdir, ignore_errors=True)
2594 fetcher.unpack(self.d.getVar('WORKDIR'))
2595
2596 def test_lfs_enabled_not_installed_but_not_needed(self):
2597 srcdir = os.path.join(self.tempdir, "emptygit")
2598 bb.utils.mkdirhier(srcdir)
2599 self.git_init(srcdir)
2600 self.commit_file("test", "test content", cwd=srcdir)
2601
2602 uri = 'git://%s;protocol=file;lfs=1;branch=master' % srcdir
2603 self.d.setVar('SRC_URI', uri)
2604
2605 # Careful: suppress initial attempt at downloading
2606 fetcher, ud = self.fetch(uri=None, download=False)
2607
2608 # It shouldnt't matter that git-lfs cannot be found as the repository configuration does not
2609 # specify any LFS filters.
2610 with unittest.mock.patch("shutil.which", return_value=None):
2611 fetcher.download()
2612 shutil.rmtree(self.gitdir, ignore_errors=True)
2613 fetcher.unpack(self.d.getVar('WORKDIR'))
2614
2615class GitURLWithSpacesTest(FetcherTest):
2616 test_git_urls = {
2617 "git://tfs-example.org:22/tfs/example%20path/example.git;branch=master" : {
2618 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git;branch=master',
2619 'repo_url': 'git://tfs-example.org:22/tfs/example%20path/example.git',
2620 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git',
2621 'path': '/tfs/example path/example.git'
2622 },
2623 "git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master" : {
2624 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master',
2625 'repo_url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git',
2626 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git',
2627 'path': '/tfs/example path/example repo.git'
2628 }
2629 }
2630
2631 def test_urls(self):
2632
2633 # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo
2634 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
2635
2636 for test_git_url, ref in self.test_git_urls.items():
2637
2638 fetcher = bb.fetch.Fetch([test_git_url], self.d)
2639 ud = fetcher.ud[fetcher.urls[0]]
2640
2641 self.assertEqual(ud.url, ref['url'])
2642 self.assertEqual(ud.path, ref['path'])
2643 self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname']))
2644 self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname']))
2645 self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock'))
2646 self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname']))
2647 self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz'))
2648 self.assertEqual(ud.method._get_repo_url(ud), ref['repo_url'])
2649
2650
2651class FetchLocallyMissingTagFromRemote(FetcherTest):
2652 def setUp(self):
2653 FetcherTest.setUp(self)
2654 self.gitdir = os.path.join(self.tempdir, 'git')
2655 self.srcdir = os.path.join(self.tempdir, 'gitsource')
2656
2657 bb.utils.mkdirhier(self.srcdir)
2658 self.git_init(cwd=self.srcdir)
2659 self.d.setVar('WORKDIR', self.tempdir)
2660 self.d.setVar('S', self.gitdir)
2661
2662 uri = 'git://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
2663 self.d.setVar('SRC_URI', uri)
2664
2665 open(os.path.join(self.srcdir, 'dummyfile'), 'w').close()
2666 self.git(['add', 'dummyfile'], self.srcdir)
2667 self.git(['commit', '-m', 'dummymsg', 'dummyfile'], self.srcdir)
2668
2669 def _fetch_and_unpack(self, uri_to_fetch):
2670 fetcher = bb.fetch2.Fetch([uri_to_fetch], self.d)
2671 fetcher.download()
2672 fetcher.unpack(self.d.getVar('WORKDIR'))
2673
2674 def test_tag_present_in_remote_but_not_local(self):
2675 # fetch a repo that has no tag in it
2676 # then add a tag to this repo, and fetch it again, without
2677 # changing SRC_REV, but by adding ';tag=tag1` to SRC_URI
2678 # the new tag should be fetched and unpacked
2679 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip()
2680 self.d.setVar('SRCREV', srcrev)
2681 src_uri = self.d.getVar('SRC_URI')
2682 self._fetch_and_unpack(src_uri)
2683
2684 self.git('tag -m -a tag1', cwd=self.srcdir)
2685
2686 src_uri = '%s;tag=tag1' % self.d.getVar('SRC_URI').split()[0]
2687 self.d.setVar('SRC_URI', src_uri)
2688 self._fetch_and_unpack(src_uri)
2689
2690 output = self.git('log --pretty=oneline -n 1 refs/tags/tag1', cwd=self.gitdir)
2691 assert "fatal: ambiguous argument" not in output
2692
2693
2694class CrateTest(FetcherTest):
2695 @skipIfNoNetwork()
2696 def test_crate_url(self):
2697
2698 uri = "crate://crates.io/glob/0.2.11"
2699 self.d.setVar('SRC_URI', uri)
2700
2701 uris = self.d.getVar('SRC_URI').split()
2702 d = self.d
2703
2704 fetcher = bb.fetch2.Fetch(uris, self.d)
2705 ud = fetcher.ud[fetcher.urls[0]]
2706
2707 self.assertIn("name", ud.parm)
2708 self.assertEqual(ud.parm["name"], "glob-0.2.11")
2709 self.assertIn("downloadfilename", ud.parm)
2710 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
2711
2712 fetcher.download()
2713 fetcher.unpack(self.tempdir)
2714 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked'])
2715 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done'])
2716 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json"))
2717 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs"))
2718
2719 @skipIfNoNetwork()
2720 def test_crate_url_matching_recipe(self):
2721
2722 self.d.setVar('BP', 'glob-0.2.11')
2723
2724 uri = "crate://crates.io/glob/0.2.11"
2725 self.d.setVar('SRC_URI', uri)
2726
2727 uris = self.d.getVar('SRC_URI').split()
2728 d = self.d
2729
2730 fetcher = bb.fetch2.Fetch(uris, self.d)
2731 ud = fetcher.ud[fetcher.urls[0]]
2732
2733 self.assertIn("name", ud.parm)
2734 self.assertEqual(ud.parm["name"], "glob-0.2.11")
2735 self.assertIn("downloadfilename", ud.parm)
2736 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
2737
2738 fetcher.download()
2739 fetcher.unpack(self.tempdir)
2740 self.assertEqual(sorted(os.listdir(self.tempdir)), ['download', 'glob-0.2.11', 'unpacked'])
2741 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done'])
2742 self.assertTrue(os.path.exists(self.tempdir + "/glob-0.2.11/src/lib.rs"))
2743
2744 @skipIfNoNetwork()
2745 def test_crate_url_params(self):
2746
2747 uri = "crate://crates.io/aho-corasick/0.7.20;name=aho-corasick-renamed"
2748 self.d.setVar('SRC_URI', uri)
2749
2750 uris = self.d.getVar('SRC_URI').split()
2751 d = self.d
2752
2753 fetcher = bb.fetch2.Fetch(uris, self.d)
2754 ud = fetcher.ud[fetcher.urls[0]]
2755
2756 self.assertIn("name", ud.parm)
2757 self.assertEqual(ud.parm["name"], "aho-corasick-renamed")
2758 self.assertIn("downloadfilename", ud.parm)
2759 self.assertEqual(ud.parm["downloadfilename"], "aho-corasick-0.7.20.crate")
2760
2761 fetcher.download()
2762 fetcher.unpack(self.tempdir)
2763 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked'])
2764 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['aho-corasick-0.7.20.crate', 'aho-corasick-0.7.20.crate.done'])
2765 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/.cargo-checksum.json"))
2766 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/src/lib.rs"))
2767
2768 @skipIfNoNetwork()
2769 def test_crate_url_multi(self):
2770
2771 uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35"
2772 self.d.setVar('SRC_URI', uri)
2773
2774 uris = self.d.getVar('SRC_URI').split()
2775 d = self.d
2776
2777 fetcher = bb.fetch2.Fetch(uris, self.d)
2778 ud = fetcher.ud[fetcher.urls[0]]
2779
2780 self.assertIn("name", ud.parm)
2781 self.assertEqual(ud.parm["name"], "glob-0.2.11")
2782 self.assertIn("downloadfilename", ud.parm)
2783 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
2784
2785 ud = fetcher.ud[fetcher.urls[1]]
2786 self.assertIn("name", ud.parm)
2787 self.assertEqual(ud.parm["name"], "time-0.1.35")
2788 self.assertIn("downloadfilename", ud.parm)
2789 self.assertEqual(ud.parm["downloadfilename"], "time-0.1.35.crate")
2790
2791 fetcher.download()
2792 fetcher.unpack(self.tempdir)
2793 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked'])
2794 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done', 'time-0.1.35.crate', 'time-0.1.35.crate.done'])
2795 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json"))
2796 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs"))
2797 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json"))
2798 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs"))
2799
2800 @skipIfNoNetwork()
2801 def test_crate_incorrect_cksum(self):
2802 uri = "crate://crates.io/aho-corasick/0.7.20"
2803 self.d.setVar('SRC_URI', uri)
2804 self.d.setVarFlag("SRC_URI", "aho-corasick-0.7.20.sha256sum", hashlib.sha256("Invalid".encode("utf-8")).hexdigest())
2805
2806 uris = self.d.getVar('SRC_URI').split()
2807
2808 fetcher = bb.fetch2.Fetch(uris, self.d)
2809 with self.assertRaisesRegex(bb.fetch2.FetchError, "Fetcher failure for URL"):
2810 fetcher.download()
2811
2812class NPMTest(FetcherTest):
2813 def skipIfNoNpm():
2814 if not shutil.which('npm'):
2815 return unittest.skip('npm not installed')
2816 return lambda f: f
2817
2818 @skipIfNoNpm()
2819 @skipIfNoNetwork()
2820 def test_npm(self):
2821 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
2822 fetcher = bb.fetch.Fetch(urls, self.d)
2823 ud = fetcher.ud[fetcher.urls[0]]
2824 fetcher.download()
2825 self.assertTrue(os.path.exists(ud.localpath))
2826 self.assertTrue(os.path.exists(ud.localpath + '.done'))
2827 self.assertTrue(os.path.exists(ud.resolvefile))
2828 fetcher.unpack(self.unpackdir)
2829 unpackdir = os.path.join(self.unpackdir, 'npm')
2830 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2831
2832 @skipIfNoNpm()
2833 @skipIfNoNetwork()
2834 def test_npm_bad_checksum(self):
2835 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
2836 # Fetch once to get a tarball
2837 fetcher = bb.fetch.Fetch(urls, self.d)
2838 ud = fetcher.ud[fetcher.urls[0]]
2839 fetcher.download()
2840 self.assertTrue(os.path.exists(ud.localpath))
2841 # Modify the tarball
2842 bad = b'bad checksum'
2843 with open(ud.localpath, 'wb') as f:
2844 f.write(bad)
2845 # Verify that the tarball is fetched again
2846 fetcher.download()
2847 badsum = hashlib.sha512(bad).hexdigest()
2848 self.assertTrue(os.path.exists(ud.localpath + '_bad-checksum_' + badsum))
2849 self.assertTrue(os.path.exists(ud.localpath))
2850
2851 @skipIfNoNpm()
2852 @skipIfNoNetwork()
2853 def test_npm_premirrors(self):
2854 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
2855 # Fetch once to get a tarball
2856 fetcher = bb.fetch.Fetch(urls, self.d)
2857 ud = fetcher.ud[fetcher.urls[0]]
2858 fetcher.download()
2859 self.assertTrue(os.path.exists(ud.localpath))
2860
2861 # Setup the mirror by renaming the download directory
2862 mirrordir = os.path.join(self.tempdir, 'mirror')
2863 bb.utils.rename(self.dldir, mirrordir)
2864 os.mkdir(self.dldir)
2865
2866 # Configure the premirror to be used
2867 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/npm2' % mirrordir)
2868 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
2869
2870 # Fetch again
2871 self.assertFalse(os.path.exists(ud.localpath))
2872 # The npm fetcher doesn't handle that the .resolved file disappears
2873 # while the fetcher object exists, which it does when we rename the
2874 # download directory to "mirror" above. Thus we need a new fetcher to go
2875 # with the now empty download directory.
2876 fetcher = bb.fetch.Fetch(urls, self.d)
2877 ud = fetcher.ud[fetcher.urls[0]]
2878 fetcher.download()
2879 self.assertTrue(os.path.exists(ud.localpath))
2880
2881 @skipIfNoNpm()
2882 @skipIfNoNetwork()
2883 def test_npm_premirrors_with_specified_filename(self):
2884 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
2885 # Fetch once to get a tarball
2886 fetcher = bb.fetch.Fetch(urls, self.d)
2887 ud = fetcher.ud[fetcher.urls[0]]
2888 fetcher.download()
2889 self.assertTrue(os.path.exists(ud.localpath))
2890 # Setup the mirror
2891 mirrordir = os.path.join(self.tempdir, 'mirror')
2892 bb.utils.mkdirhier(mirrordir)
2893 mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath))
2894 os.replace(ud.localpath, mirrorfilename)
2895 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s' % mirrorfilename)
2896 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
2897 # Fetch again
2898 self.assertFalse(os.path.exists(ud.localpath))
2899 fetcher.download()
2900 self.assertTrue(os.path.exists(ud.localpath))
2901
2902 @skipIfNoNpm()
2903 @skipIfNoNetwork()
2904 def test_npm_mirrors(self):
2905 # Fetch once to get a tarball
2906 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
2907 fetcher = bb.fetch.Fetch(urls, self.d)
2908 ud = fetcher.ud[fetcher.urls[0]]
2909 fetcher.download()
2910 self.assertTrue(os.path.exists(ud.localpath))
2911 # Setup the mirror
2912 mirrordir = os.path.join(self.tempdir, 'mirror')
2913 bb.utils.mkdirhier(mirrordir)
2914 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
2915 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir)
2916 # Update the resolved url to an invalid url
2917 with open(ud.resolvefile, 'r') as f:
2918 url = f.read()
2919 uri = URI(url)
2920 uri.path = '/invalid'
2921 with open(ud.resolvefile, 'w') as f:
2922 f.write(str(uri))
2923 # Fetch again
2924 self.assertFalse(os.path.exists(ud.localpath))
2925 fetcher.download()
2926 self.assertTrue(os.path.exists(ud.localpath))
2927
2928 @skipIfNoNpm()
2929 @skipIfNoNetwork()
2930 def test_npm_destsuffix_downloadfilename(self):
2931 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz']
2932 fetcher = bb.fetch.Fetch(urls, self.d)
2933 fetcher.download()
2934 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'foo-bar.tgz')))
2935 fetcher.unpack(self.unpackdir)
2936 unpackdir = os.path.join(self.unpackdir, 'foo', 'bar')
2937 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2938
2939 def test_npm_no_network_no_tarball(self):
2940 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
2941 self.d.setVar('BB_NO_NETWORK', '1')
2942 fetcher = bb.fetch.Fetch(urls, self.d)
2943 with self.assertRaises(bb.fetch2.NetworkAccess):
2944 fetcher.download()
2945
2946 @skipIfNoNpm()
2947 @skipIfNoNetwork()
2948 def test_npm_no_network_with_tarball(self):
2949 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
2950 # Fetch once to get a tarball
2951 fetcher = bb.fetch.Fetch(urls, self.d)
2952 fetcher.download()
2953 # Disable network access
2954 self.d.setVar('BB_NO_NETWORK', '1')
2955 # Fetch again
2956 fetcher.download()
2957 fetcher.unpack(self.unpackdir)
2958 unpackdir = os.path.join(self.unpackdir, 'npm')
2959 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2960
2961 @skipIfNoNpm()
2962 @skipIfNoNetwork()
2963 def test_npm_registry_alternate(self):
2964 urls = ['npm://skimdb.npmjs.com;package=@savoirfairelinux/node-server-example;version=1.0.0']
2965 fetcher = bb.fetch.Fetch(urls, self.d)
2966 fetcher.download()
2967 fetcher.unpack(self.unpackdir)
2968 unpackdir = os.path.join(self.unpackdir, 'npm')
2969 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2970
2971 @skipIfNoNpm()
2972 @skipIfNoNetwork()
2973 def test_npm_version_latest(self):
2974 url = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest']
2975 fetcher = bb.fetch.Fetch(url, self.d)
2976 fetcher.download()
2977 fetcher.unpack(self.unpackdir)
2978 unpackdir = os.path.join(self.unpackdir, 'npm')
2979 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2980
2981 @skipIfNoNpm()
2982 @skipIfNoNetwork()
2983 def test_npm_registry_invalid(self):
2984 urls = ['npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
2985 fetcher = bb.fetch.Fetch(urls, self.d)
2986 with self.assertRaises(bb.fetch2.FetchError):
2987 fetcher.download()
2988
2989 @skipIfNoNpm()
2990 @skipIfNoNetwork()
2991 def test_npm_package_invalid(self):
2992 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0']
2993 fetcher = bb.fetch.Fetch(urls, self.d)
2994 with self.assertRaises(bb.fetch2.FetchError):
2995 fetcher.download()
2996
2997 @skipIfNoNpm()
2998 @skipIfNoNetwork()
2999 def test_npm_version_invalid(self):
3000 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid']
3001 with self.assertRaises(bb.fetch2.ParameterError):
3002 fetcher = bb.fetch.Fetch(urls, self.d)
3003
3004 @skipIfNoNpm()
3005 @skipIfNoNetwork()
3006 def test_npm_registry_none(self):
3007 urls = ['npm://;package=@savoirfairelinux/node-server-example;version=1.0.0']
3008 with self.assertRaises(bb.fetch2.MalformedUrl):
3009 fetcher = bb.fetch.Fetch(urls, self.d)
3010
3011 @skipIfNoNpm()
3012 @skipIfNoNetwork()
3013 def test_npm_package_none(self):
3014 urls = ['npm://registry.npmjs.org;version=1.0.0']
3015 with self.assertRaises(bb.fetch2.MissingParameterError):
3016 fetcher = bb.fetch.Fetch(urls, self.d)
3017
3018 @skipIfNoNpm()
3019 @skipIfNoNetwork()
3020 def test_npm_version_none(self):
3021 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example']
3022 with self.assertRaises(bb.fetch2.MissingParameterError):
3023 fetcher = bb.fetch.Fetch(urls, self.d)
3024
3025 def create_shrinkwrap_file(self, data):
3026 import json
3027 datadir = os.path.join(self.tempdir, 'data')
3028 swfile = os.path.join(datadir, 'npm-shrinkwrap.json')
3029 bb.utils.mkdirhier(datadir)
3030 with open(swfile, 'w') as f:
3031 json.dump(data, f)
3032 return swfile
3033
3034 @skipIfNoNetwork()
3035 def test_npmsw(self):
3036 swfile = self.create_shrinkwrap_file({
3037 'packages': {
3038 'node_modules/array-flatten': {
3039 'version': '1.1.1',
3040 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
3041 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=',
3042 'dependencies': {
3043 'content-type': "1.0.4"
3044 }
3045 },
3046 'node_modules/array-flatten/node_modules/content-type': {
3047 'version': '1.0.4',
3048 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
3049 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
3050 'dependencies': {
3051 'cookie': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09'
3052 }
3053 },
3054 'node_modules/array-flatten/node_modules/content-type/node_modules/cookie': {
3055 'resolved': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09'
3056 }
3057 }
3058 })
3059 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
3060 fetcher.download()
3061 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
3062 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
3063 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git')))
3064 fetcher.unpack(self.unpackdir)
3065 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm-shrinkwrap.json')))
3066 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'package.json')))
3067 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json')))
3068 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json')))
3069
3070 @skipIfNoNetwork()
3071 def test_npmsw_git(self):
3072 swfile = self.create_shrinkwrap_file({
3073 'packages': {
3074 'node_modules/cookie': {
3075 'resolved': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09'
3076 }
3077 }
3078 })
3079 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
3080 fetcher.download()
3081 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git')))
3082
3083 @skipIfNoNetwork()
3084 def test_npmsw_dev(self):
3085 swfile = self.create_shrinkwrap_file({
3086 'packages': {
3087 'node_modules/array-flatten': {
3088 'version': '1.1.1',
3089 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
3090 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
3091 },
3092 'node_modules/content-type': {
3093 'version': '1.0.4',
3094 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
3095 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
3096 'dev': True
3097 }
3098 }
3099 })
3100 # Fetch with dev disabled
3101 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
3102 fetcher.download()
3103 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
3104 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
3105 # Fetch with dev enabled
3106 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d)
3107 fetcher.download()
3108 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
3109 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
3110
3111 @skipIfNoNetwork()
3112 def test_npmsw_destsuffix(self):
3113 swfile = self.create_shrinkwrap_file({
3114 'packages': {
3115 'node_modules/array-flatten': {
3116 'version': '1.1.1',
3117 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
3118 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
3119 }
3120 }
3121 })
3122 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d)
3123 fetcher.download()
3124 fetcher.unpack(self.unpackdir)
3125 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json')))
3126
3127 def test_npmsw_no_network_no_tarball(self):
3128 swfile = self.create_shrinkwrap_file({
3129 'packages': {
3130 'node_modules/array-flatten': {
3131 'version': '1.1.1',
3132 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
3133 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
3134 }
3135 }
3136 })
3137 self.d.setVar('BB_NO_NETWORK', '1')
3138 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
3139 with self.assertRaises(bb.fetch2.NetworkAccess):
3140 fetcher.download()
3141
3142 @skipIfNoNpm()
3143 @skipIfNoNetwork()
3144 def test_npmsw_no_network_with_tarball(self):
3145 # Fetch once to get a tarball
3146 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
3147 fetcher.download()
3148 # Disable network access
3149 self.d.setVar('BB_NO_NETWORK', '1')
3150 # Fetch again
3151 swfile = self.create_shrinkwrap_file({
3152 'packages': {
3153 'node_modules/array-flatten': {
3154 'version': '1.1.1',
3155 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
3156 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
3157 }
3158 }
3159 })
3160 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
3161 fetcher.download()
3162 fetcher.unpack(self.unpackdir)
3163 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'package.json')))
3164
3165 @skipIfNoNetwork()
3166 def test_npmsw_npm_reusability(self):
3167 # Fetch once with npmsw
3168 swfile = self.create_shrinkwrap_file({
3169 'packages': {
3170 'node_modules/array-flatten': {
3171 'version': '1.1.1',
3172 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
3173 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
3174 }
3175 }
3176 })
3177 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
3178 fetcher.download()
3179 # Disable network access
3180 self.d.setVar('BB_NO_NETWORK', '1')
3181 # Fetch again with npm
3182 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
3183 fetcher.download()
3184 fetcher.unpack(self.unpackdir)
3185 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json')))
3186
3187 @skipIfNoNetwork()
3188 def test_npmsw_bad_checksum(self):
3189 # Try to fetch with bad checksum
3190 swfile = self.create_shrinkwrap_file({
3191 'packages': {
3192 'node_modules/array-flatten': {
3193 'version': '1.1.1',
3194 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
3195 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg='
3196 }
3197 }
3198 })
3199 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
3200 with self.assertRaises(bb.fetch2.FetchError):
3201 fetcher.download()
3202 # Fetch correctly to get a tarball
3203 swfile = self.create_shrinkwrap_file({
3204 'packages': {
3205 'node_modules/array-flatten': {
3206 'version': '1.1.1',
3207 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
3208 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
3209 }
3210 }
3211 })
3212 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
3213 fetcher.download()
3214 localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')
3215 self.assertTrue(os.path.exists(localpath))
3216 # Modify the tarball
3217 bad = b'bad checksum'
3218 with open(localpath, 'wb') as f:
3219 f.write(bad)
3220 # Verify that the tarball is fetched again
3221 fetcher.download()
3222 badsum = hashlib.sha1(bad).hexdigest()
3223 self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum))
3224 self.assertTrue(os.path.exists(localpath))
3225
3226 @skipIfNoNpm()
3227 @skipIfNoNetwork()
3228 def test_npmsw_premirrors(self):
3229 # Fetch once to get a tarball
3230 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
3231 ud = fetcher.ud[fetcher.urls[0]]
3232 fetcher.download()
3233 self.assertTrue(os.path.exists(ud.localpath))
3234 # Setup the mirror
3235 mirrordir = os.path.join(self.tempdir, 'mirror')
3236 bb.utils.mkdirhier(mirrordir)
3237 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
3238 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir)
3239 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
3240 # Fetch again
3241 self.assertFalse(os.path.exists(ud.localpath))
3242 swfile = self.create_shrinkwrap_file({
3243 'packages': {
3244 'node_modules/array-flatten': {
3245 'version': '1.1.1',
3246 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
3247 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
3248 }
3249 }
3250 })
3251 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
3252 fetcher.download()
3253 self.assertTrue(os.path.exists(ud.localpath))
3254
3255 @skipIfNoNpm()
3256 @skipIfNoNetwork()
3257 def test_npmsw_mirrors(self):
3258 # Fetch once to get a tarball
3259 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
3260 ud = fetcher.ud[fetcher.urls[0]]
3261 fetcher.download()
3262 self.assertTrue(os.path.exists(ud.localpath))
3263 # Setup the mirror
3264 mirrordir = os.path.join(self.tempdir, 'mirror')
3265 bb.utils.mkdirhier(mirrordir)
3266 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
3267 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir)
3268 # Fetch again with invalid url
3269 self.assertFalse(os.path.exists(ud.localpath))
3270 swfile = self.create_shrinkwrap_file({
3271 'packages': {
3272 'node_modules/array-flatten': {
3273 'version': '1.1.1',
3274 'resolved': 'https://invalid',
3275 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
3276 }
3277 }
3278 })
3279 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
3280 fetcher.download()
3281 self.assertTrue(os.path.exists(ud.localpath))
3282
3283 @skipIfNoNetwork()
3284 def test_npmsw_bundled(self):
3285 swfile = self.create_shrinkwrap_file({
3286 'packages': {
3287 'node_modules/array-flatten': {
3288 'version': '1.1.1',
3289 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
3290 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
3291 },
3292 'node_modules/content-type': {
3293 'version': '1.0.4',
3294 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
3295 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
3296 'inBundle': True
3297 }
3298 }
3299 })
3300 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
3301 fetcher.download()
3302 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
3303 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
3304
3305class GitSharedTest(FetcherTest):
3306 def setUp(self):
3307 super(GitSharedTest, self).setUp()
3308 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
3309 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
3310 self.d.setVar("__BBSRCREV_SEEN", "1")
3311
3312 @skipIfNoNetwork()
3313 def test_shared_unpack(self):
3314 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3315
3316 fetcher.download()
3317 fetcher.unpack(self.unpackdir)
3318 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
3319 self.assertTrue(os.path.exists(alt))
3320
3321 @skipIfNoNetwork()
3322 def test_noshared_unpack(self):
3323 self.d.setVar('BB_GIT_NOSHARED', '1')
3324 self.unpackdir += '_noshared'
3325 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3326
3327 fetcher.download()
3328 fetcher.unpack(self.unpackdir)
3329 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
3330 self.assertFalse(os.path.exists(alt))
3331
3332class GitTagVerificationTests(FetcherTest):
3333
3334 @skipIfNoNetwork()
3335 def test_tag_rev_match(self):
3336 # Test a url with rev= and tag= set works
3337 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.12;protocol=https;rev=5b4e20377eea8d428edf1aeb2187c18f82ca6757;tag=2.12.0"], self.d)
3338 fetcher.download()
3339 fetcher.unpack(self.unpackdir)
3340
3341 def test_annotated_tag_rev_match(self):
3342 # Test a url with rev= and tag= set works
3343 # rev is the annotated tag revision in this case
3344 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.12;protocol=https;rev=fa30183549bd09f33fd4eebf56771ca5393526a6;tag=2.12.0"], self.d)
3345 fetcher.download()
3346 fetcher.unpack(self.unpackdir)
3347
3348 @skipIfNoNetwork()
3349 def test_tag_rev_match2(self):
3350 # Test a url with SRCREV and tag= set works
3351 self.d.setVar('SRCREV', '5b4e20377eea8d428edf1aeb2187c18f82ca6757')
3352 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.12;protocol=https;tag=2.12.0"], self.d)
3353 fetcher.download()
3354 fetcher.unpack(self.unpackdir)
3355
3356 @skipIfNoNetwork()
3357 def test_tag_rev_match3(self):
3358 # Test a url with SRCREV, rev= and tag= set works
3359 self.d.setVar('SRCREV', '5b4e20377eea8d428edf1aeb2187c18f82ca6757')
3360 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.12;protocol=https;rev=5b4e20377eea8d428edf1aeb2187c18f82ca6757;tag=2.12.0"], self.d)
3361 fetcher.download()
3362 fetcher.unpack(self.unpackdir)
3363
3364 @skipIfNoNetwork()
3365 def test_tag_rev_match4(self):
3366 # Test a url with SRCREV and rev= mismatching errors
3367 self.d.setVar('SRCREV', 'bade540fc31a1c26839efd2c7785a751ce24ebfb')
3368 with self.assertRaises(bb.fetch2.FetchError):
3369 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.12;protocol=https;rev=5b4e20377eea8d428edf1aeb2187c18f82ca6757;tag=2.12.0"], self.d)
3370
3371 @skipIfNoNetwork()
3372 def test_tag_rev_match5(self):
3373 # Test a url with SRCREV, rev= and tag= set works when using shallow clones
3374 self.d.setVar('BB_GIT_SHALLOW', '1')
3375 self.d.setVar('SRCREV', '5b4e20377eea8d428edf1aeb2187c18f82ca6757')
3376 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.12;protocol=https;rev=5b4e20377eea8d428edf1aeb2187c18f82ca6757;tag=2.12.0"], self.d)
3377 fetcher.download()
3378 fetcher.unpack(self.unpackdir)
3379
3380 @skipIfNoNetwork()
3381 def test_tag_rev_match6(self):
3382 # Test a url with SRCREV, rev= and a mismatched tag= when using shallow clones
3383 self.d.setVar('BB_GIT_SHALLOW', '1')
3384 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.12;protocol=https;rev=5b4e20377eea8d428edf1aeb2187c18f82ca6757;tag=2.8.0"], self.d)
3385 fetcher.download()
3386 with self.assertRaises(bb.fetch2.FetchError):
3387 fetcher.unpack(self.unpackdir)
3388
3389 @skipIfNoNetwork()
3390 def test_tag_rev_match7(self):
3391 # Test a url with SRCREV, rev= and a mismatched tag=
3392 self.d.setVar('SRCREV', '5b4e20377eea8d428edf1aeb2187c18f82ca6757')
3393 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.12;protocol=https;rev=5b4e20377eea8d428edf1aeb2187c18f82ca6757;tag=2.8.0"], self.d)
3394 fetcher.download()
3395 with self.assertRaises(bb.fetch2.FetchError):
3396 fetcher.unpack(self.unpackdir)
3397
3398
3399class FetchPremirroronlyLocalTest(FetcherTest):
3400
3401 def setUp(self):
3402 super(FetchPremirroronlyLocalTest, self).setUp()
3403 self.mirrordir = os.path.join(self.tempdir, "mirrors")
3404 os.mkdir(self.mirrordir)
3405 self.reponame = "bitbake"
3406 self.gitdir = os.path.join(self.tempdir, "git", self.reponame)
3407 self.recipe_url = "git://git.fake.repo/bitbake;branch=master;protocol=https"
3408 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
3409 self.d.setVar("BB_NO_NETWORK", "1")
3410 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
3411 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz"
3412 self.mirrorfile = os.path.join(self.mirrordir, self.mirrorname)
3413 self.testfilename = "bitbake-fetch.test"
3414
3415 def make_git_repo(self):
3416 recipeurl = "git:/git.fake.repo/bitbake"
3417 os.makedirs(self.gitdir)
3418 self.git_init(cwd=self.gitdir)
3419 for i in range(0):
3420 self.git_new_commit()
3421 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir)
3422
3423 def git_new_commit(self):
3424 import random
3425 os.unlink(os.path.join(self.mirrordir, self.mirrorname))
3426 branch = self.git("branch --show-current", self.gitdir).split()
3427 with open(os.path.join(self.gitdir, self.testfilename), "w") as testfile:
3428 testfile.write("File {} from branch {}; Useless random data {}".format(self.testfilename, branch, random.random()))
3429 self.git("add {}".format(self.testfilename), self.gitdir)
3430 self.git("commit -a -m \"This random commit {} in branch {}. I'm useless.\"".format(random.random(), branch), self.gitdir)
3431 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir)
3432 return self.git("rev-parse HEAD", self.gitdir).strip()
3433
3434 def git_new_branch(self, name):
3435 self.git_new_commit()
3436 head = self.git("rev-parse HEAD", self.gitdir).strip()
3437 self.git("checkout -b {}".format(name), self.gitdir)
3438 newrev = self.git_new_commit()
3439 self.git("checkout {}".format(head), self.gitdir)
3440 return newrev
3441
3442 def test_mirror_multiple_fetches(self):
3443 self.make_git_repo()
3444 self.d.setVar("SRCREV", self.git_new_commit())
3445 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3446 fetcher.download()
3447 fetcher.unpack(self.unpackdir)
3448 ## New commit in premirror. it's not in the download_dir
3449 self.d.setVar("SRCREV", self.git_new_commit())
3450 fetcher2 = bb.fetch.Fetch([self.recipe_url], self.d)
3451 fetcher2.download()
3452 fetcher2.unpack(self.unpackdir)
3453 ## New commit in premirror. it's not in the download_dir
3454 self.d.setVar("SRCREV", self.git_new_commit())
3455 fetcher3 = bb.fetch.Fetch([self.recipe_url], self.d)
3456 fetcher3.download()
3457 fetcher3.unpack(self.unpackdir)
3458
3459
3460 def test_mirror_commit_nonexistent(self):
3461 self.make_git_repo()
3462 self.d.setVar("SRCREV", "0"*40)
3463 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3464 with self.assertRaises(bb.fetch2.NetworkAccess):
3465 fetcher.download()
3466
3467 def test_mirror_commit_exists(self):
3468 self.make_git_repo()
3469 self.d.setVar("SRCREV", self.git_new_commit())
3470 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3471 fetcher.download()
3472 fetcher.unpack(self.unpackdir)
3473
3474 def test_mirror_tarball_nonexistent(self):
3475 self.d.setVar("SRCREV", "0"*40)
3476 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3477 with self.assertRaises(bb.fetch2.NetworkAccess):
3478 fetcher.download()
3479
3480
3481class FetchPremirroronlyNetworkTest(FetcherTest):
3482
3483 def setUp(self):
3484 super(FetchPremirroronlyNetworkTest, self).setUp()
3485 self.mirrordir = os.path.join(self.tempdir, "mirrors")
3486 os.mkdir(self.mirrordir)
3487 self.reponame = "fstests"
3488 self.clonedir = os.path.join(self.tempdir, "git")
3489 self.gitdir = os.path.join(self.tempdir, "git", "{}.git".format(self.reponame))
3490 self.recipe_url = "git://git.yoctoproject.org/fstests;protocol=https;branch=master"
3491 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
3492 self.d.setVar("BB_NO_NETWORK", "0")
3493 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
3494
3495 def make_git_repo(self):
3496 self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz"
3497 os.makedirs(self.clonedir)
3498 self.git("clone --bare {}".format(self.recipe_url), self.clonedir)
3499 self.git("update-ref HEAD 15413486df1f5a5b5af699b6f3ba5f0984e52a9f", self.gitdir)
3500 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir)
3501 shutil.rmtree(self.clonedir)
3502
3503 @skipIfNoNetwork()
3504 def test_mirror_tarball_updated(self):
3505 self.make_git_repo()
3506 ## Upstream commit is in the mirror
3507 self.d.setVar("SRCREV", "15413486df1f5a5b5af699b6f3ba5f0984e52a9f")
3508 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3509 fetcher.download()
3510
3511 @skipIfNoNetwork()
3512 def test_mirror_tarball_outdated(self):
3513 self.make_git_repo()
3514 ## Upstream commit not in the mirror
3515 self.d.setVar("SRCREV", "49d65d53c2bf558ae6e9185af0f3af7b79d255ec")
3516 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3517 with self.assertRaises(bb.fetch2.NetworkAccess):
3518 fetcher.download()
3519
3520class FetchPremirroronlyMercurialTest(FetcherTest):
3521 """ Test for premirrors with mercurial repos
3522 the test covers also basic hg:// clone (see fetch_and_create_tarball
3523 """
3524 def skipIfNoHg():
3525 if not shutil.which('hg'):
3526 return unittest.skip('Mercurial not installed')
3527 return lambda f: f
3528
3529 def setUp(self):
3530 super(FetchPremirroronlyMercurialTest, self).setUp()
3531 self.mirrordir = os.path.join(self.tempdir, "mirrors")
3532 os.mkdir(self.mirrordir)
3533 self.reponame = "libgnt"
3534 self.clonedir = os.path.join(self.tempdir, "hg")
3535 self.recipe_url = "hg://keep.imfreedom.org/libgnt;module=libgnt"
3536 self.d.setVar("SRCREV", "53e8b422faaf")
3537 self.mirrorname = "hg_libgnt_keep.imfreedom.org_.libgnt.tar.gz"
3538
3539 def fetch_and_create_tarball(self):
3540 """
3541 Ask bitbake to download repo and prepare mirror tarball for us
3542 """
3543 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
3544 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3545 fetcher.download()
3546 mirrorfile = os.path.join(self.d.getVar("DL_DIR"), self.mirrorname)
3547 self.assertTrue(os.path.exists(mirrorfile), "Mirror tarball {} has not been created".format(mirrorfile))
3548 ## moving tarball to mirror directory
3549 os.rename(mirrorfile, os.path.join(self.mirrordir, self.mirrorname))
3550 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "0")
3551
3552
3553 @skipIfNoNetwork()
3554 @skipIfNoHg()
3555 def test_premirror_mercurial(self):
3556 self.fetch_and_create_tarball()
3557 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
3558 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
3559 self.d.setVar("BB_NO_NETWORK", "1")
3560 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3561 fetcher.download()
3562
3563class FetchPremirroronlyBrokenTarball(FetcherTest):
3564
3565 def setUp(self):
3566 super(FetchPremirroronlyBrokenTarball, self).setUp()
3567 self.mirrordir = os.path.join(self.tempdir, "mirrors")
3568 os.mkdir(self.mirrordir)
3569 self.reponame = "bitbake"
3570 self.gitdir = os.path.join(self.tempdir, "git", self.reponame)
3571 self.recipe_url = "git://git.fake.repo/bitbake;protocol=https;branch=master"
3572 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
3573 self.d.setVar("BB_NO_NETWORK", "1")
3574 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
3575 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz"
3576 with open(os.path.join(self.mirrordir, self.mirrorname), 'w') as targz:
3577 targz.write("This is not tar.gz file!")
3578
3579 def test_mirror_broken_download(self):
3580 self.d.setVar("SRCREV", "0"*40)
3581 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3582 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs() as logs:
3583 fetcher.download()
3584 output = "".join(logs.output)
3585 self.assertFalse(" not a git repository (or any parent up to mount point /)" in output)
3586
3587class GoModTest(FetcherTest):
3588
3589 @skipIfNoNetwork()
3590 def test_gomod_url(self):
3591 urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;'
3592 'sha256sum=9bb69aea32f1d59711701f9562d66432c9c0374205e5009d1d1a62f03fb4fdad']
3593
3594 fetcher = bb.fetch2.Fetch(urls, self.d)
3595 ud = fetcher.ud[urls[0]]
3596 self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.zip')
3597 self.assertEqual(ud.parm['downloadfilename'], 'github.com.Azure.azure-sdk-for-go.sdk.storage.azblob@v1.0.0.zip')
3598 self.assertEqual(ud.parm['name'], 'github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0')
3599
3600 fetcher.download()
3601 fetcher.unpack(self.unpackdir)
3602 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
3603 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip')))
3604 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')))
3605 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')),
3606 '7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873')
3607
3608 @skipIfNoNetwork()
3609 def test_gomod_url_go_mod_only(self):
3610 urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;mod=1;'
3611 'sha256sum=7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873']
3612
3613 fetcher = bb.fetch2.Fetch(urls, self.d)
3614 ud = fetcher.ud[urls[0]]
3615 self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.mod')
3616 self.assertEqual(ud.parm['downloadfilename'], 'github.com.Azure.azure-sdk-for-go.sdk.storage.azblob@v1.0.0.mod')
3617 self.assertEqual(ud.parm['name'], 'github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0')
3618
3619 fetcher.download()
3620 fetcher.unpack(self.unpackdir)
3621 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
3622 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')))
3623
3624 @skipIfNoNetwork()
3625 def test_gomod_url_sha256sum_varflag(self):
3626 urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0']
3627 self.d.setVarFlag('SRC_URI', 'gopkg.in/ini.v1@v1.67.0.sha256sum', 'bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6')
3628
3629 fetcher = bb.fetch2.Fetch(urls, self.d)
3630 ud = fetcher.ud[urls[0]]
3631 self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip')
3632 self.assertEqual(ud.parm['downloadfilename'], 'gopkg.in.ini.v1@v1.67.0.zip')
3633 self.assertEqual(ud.parm['name'], 'gopkg.in/ini.v1@v1.67.0')
3634
3635 fetcher.download()
3636 fetcher.unpack(self.unpackdir)
3637 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
3638 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
3639 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
3640 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')),
3641 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1')
3642
3643 @skipIfNoNetwork()
3644 def test_gomod_url_no_go_mod_in_module(self):
3645 urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0;'
3646 'sha256sum=bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6']
3647
3648 fetcher = bb.fetch2.Fetch(urls, self.d)
3649 ud = fetcher.ud[urls[0]]
3650 self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip')
3651 self.assertEqual(ud.parm['downloadfilename'], 'gopkg.in.ini.v1@v1.67.0.zip')
3652 self.assertEqual(ud.parm['name'], 'gopkg.in/ini.v1@v1.67.0')
3653
3654 fetcher.download()
3655 fetcher.unpack(self.unpackdir)
3656 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
3657 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
3658 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
3659 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')),
3660 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1')
3661
3662 @skipIfNoNetwork()
3663 def test_gomod_url_host_only(self):
3664 urls = ['gomod://go.opencensus.io;version=v0.24.0;'
3665 'sha256sum=203a767d7f8e7c1ebe5588220ad168d1e15b14ae70a636de7ca9a4a88a7e0d0c']
3666
3667 fetcher = bb.fetch2.Fetch(urls, self.d)
3668 ud = fetcher.ud[urls[0]]
3669 self.assertEqual(ud.url, 'https://proxy.golang.org/go.opencensus.io/%40v/v0.24.0.zip')
3670 self.assertEqual(ud.parm['downloadfilename'], 'go.opencensus.io@v0.24.0.zip')
3671 self.assertEqual(ud.parm['name'], 'go.opencensus.io@v0.24.0')
3672
3673 fetcher.download()
3674 fetcher.unpack(self.unpackdir)
3675 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
3676 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.zip')))
3677 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')))
3678 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')),
3679 '0dc9ccc660ad21cebaffd548f2cc6efa27891c68b4fbc1f8a3893b00f1acec96')
3680
3681class GoModGitTest(FetcherTest):
3682
3683 @skipIfNoNetwork()
3684 def test_gomodgit_url_repo(self):
3685 urls = ['gomodgit://golang.org/x/net;version=v0.9.0;'
3686 'repo=go.googlesource.com/net;'
3687 'srcrev=694cff8668bac64e0864b552bffc280cd27f21b1']
3688
3689 fetcher = bb.fetch2.Fetch(urls, self.d)
3690 ud = fetcher.ud[urls[0]]
3691 self.assertEqual(ud.host, 'go.googlesource.com')
3692 self.assertEqual(ud.path, '/net')
3693 self.assertEqual(ud.name, 'golang.org/x/net@v0.9.0')
3694 self.assertEqual(self.d.getVar('SRCREV_golang.org/x/net@v0.9.0'), '694cff8668bac64e0864b552bffc280cd27f21b1')
3695
3696 fetcher.download()
3697 self.assertTrue(os.path.exists(ud.localpath))
3698
3699 fetcher.unpack(self.unpackdir)
3700 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
3701 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'ed42bd05533fd84ae290a5d33ebd3695a0a2b06131beebd5450825bee8603aca')))
3702 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
3703 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.zip')))
3704 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod')))
3705 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod')),
3706 'c5d6851ede50ec1c001afb763040194b68961bf06997e2605e8bf06dcd2aeb2e')
3707
3708 @skipIfNoNetwork()
3709 def test_gomodgit_url_subdir(self):
3710 urls = ['gomodgit://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;'
3711 'repo=github.com/Azure/azure-sdk-for-go;subdir=sdk/storage/azblob;'
3712 'srcrev=ec928e0ed34db682b3f783d3739d1c538142e0c3']
3713
3714 fetcher = bb.fetch2.Fetch(urls, self.d)
3715 ud = fetcher.ud[urls[0]]
3716 self.assertEqual(ud.host, 'github.com')
3717 self.assertEqual(ud.path, '/Azure/azure-sdk-for-go')
3718 self.assertEqual(ud.parm['subpath'], 'sdk/storage/azblob')
3719 self.assertEqual(ud.name, 'github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0')
3720 self.assertEqual(self.d.getVar('SRCREV_github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'), 'ec928e0ed34db682b3f783d3739d1c538142e0c3')
3721
3722 fetcher.download()
3723 self.assertTrue(os.path.exists(ud.localpath))
3724
3725 fetcher.unpack(self.unpackdir)
3726 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
3727 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'd31d6145676ed3066ce573a8198f326dea5be45a43b3d8f41ce7787fd71d66b3')))
3728 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
3729 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip')))
3730 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')))
3731 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')),
3732 '7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873')
3733
3734 @skipIfNoNetwork()
3735 def test_gomodgit_url_srcrev_var(self):
3736 urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0']
3737 self.d.setVar('SRCREV_gopkg.in/ini.v1@v1.67.0', 'b2f570e5b5b844226bbefe6fb521d891f529a951')
3738
3739 fetcher = bb.fetch2.Fetch(urls, self.d)
3740 ud = fetcher.ud[urls[0]]
3741 self.assertEqual(ud.host, 'gopkg.in')
3742 self.assertEqual(ud.path, '/ini.v1')
3743 self.assertEqual(ud.name, 'gopkg.in/ini.v1@v1.67.0')
3744 self.assertEqual(ud.parm['srcrev'], 'b2f570e5b5b844226bbefe6fb521d891f529a951')
3745
3746 fetcher.download()
3747 fetcher.unpack(self.unpackdir)
3748 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
3749 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3')))
3750 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
3751 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
3752 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
3753 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')),
3754 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1')
3755
3756 @skipIfNoNetwork()
3757 def test_gomodgit_url_no_go_mod_in_module(self):
3758 urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0;'
3759 'srcrev=b2f570e5b5b844226bbefe6fb521d891f529a951']
3760
3761 fetcher = bb.fetch2.Fetch(urls, self.d)
3762 ud = fetcher.ud[urls[0]]
3763 self.assertEqual(ud.host, 'gopkg.in')
3764 self.assertEqual(ud.path, '/ini.v1')
3765 self.assertEqual(ud.name, 'gopkg.in/ini.v1@v1.67.0')
3766 self.assertEqual(self.d.getVar('SRCREV_gopkg.in/ini.v1@v1.67.0'), 'b2f570e5b5b844226bbefe6fb521d891f529a951')
3767
3768 fetcher.download()
3769 fetcher.unpack(self.unpackdir)
3770 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
3771 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3')))
3772 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
3773 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
3774 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
3775 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')),
3776 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1')
3777
3778 @skipIfNoNetwork()
3779 def test_gomodgit_url_host_only(self):
3780 urls = ['gomodgit://go.opencensus.io;version=v0.24.0;'
3781 'repo=github.com/census-instrumentation/opencensus-go;'
3782 'srcrev=b1a01ee95db0e690d91d7193d037447816fae4c5']
3783
3784 fetcher = bb.fetch2.Fetch(urls, self.d)
3785 ud = fetcher.ud[urls[0]]
3786 self.assertEqual(ud.host, 'github.com')
3787 self.assertEqual(ud.path, '/census-instrumentation/opencensus-go')
3788 self.assertEqual(ud.name, 'go.opencensus.io@v0.24.0')
3789 self.assertEqual(self.d.getVar('SRCREV_go.opencensus.io@v0.24.0'), 'b1a01ee95db0e690d91d7193d037447816fae4c5')
3790
3791 fetcher.download()
3792 fetcher.unpack(self.unpackdir)
3793 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
3794 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'aae3ac7b2122ed3345654e6327855e9682f4a5350d63e93dbcfc51c4419df0e1')))
3795 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
3796 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.zip')))
3797 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')))
3798 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')),
3799 '0dc9ccc660ad21cebaffd548f2cc6efa27891c68b4fbc1f8a3893b00f1acec96')
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2026-01-09 23:28:19 +0000