summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/tests/fetch.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/tests/fetch.py')
-rw-r--r--bitbake/lib/bb/tests/fetch.py1197
1 files changed, 963 insertions, 234 deletions
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 7b2dac7b86..85c1f79ff3 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -6,11 +6,14 @@
6# SPDX-License-Identifier: GPL-2.0-only 6# SPDX-License-Identifier: GPL-2.0-only
7# 7#
8 8
9import contextlib
9import unittest 10import unittest
10import hashlib 11import hashlib
11import tempfile 12import tempfile
12import collections 13import collections
13import os 14import os
15import signal
16import tarfile
14from bb.fetch2 import URI 17from bb.fetch2 import URI
15from bb.fetch2 import FetchMethod 18from bb.fetch2 import FetchMethod
16import bb 19import bb
@@ -18,9 +21,28 @@ from bb.tests.support.httpserver import HTTPService
18 21
19def skipIfNoNetwork(): 22def skipIfNoNetwork():
20 if os.environ.get("BB_SKIP_NETTESTS") == "yes": 23 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
21 return unittest.skip("Network tests being skipped") 24 return unittest.skip("network test")
22 return lambda f: f 25 return lambda f: f
23 26
27class TestTimeout(Exception):
28 # Indicate to pytest that this is not a test suite
29 __test__ = False
30
31class Timeout():
32
33 def __init__(self, seconds):
34 self.seconds = seconds
35
36 def handle_timeout(self, signum, frame):
37 raise TestTimeout("Test failed: timeout reached")
38
39 def __enter__(self):
40 signal.signal(signal.SIGALRM, self.handle_timeout)
41 signal.alarm(self.seconds)
42
43 def __exit__(self, exc_type, exc_val, exc_tb):
44 signal.alarm(0)
45
24class URITest(unittest.TestCase): 46class URITest(unittest.TestCase):
25 test_uris = { 47 test_uris = {
26 "http://www.google.com/index.html" : { 48 "http://www.google.com/index.html" : {
@@ -286,6 +308,21 @@ class URITest(unittest.TestCase):
286 'params': {"someparam" : "1"}, 308 'params': {"someparam" : "1"},
287 'query': {}, 309 'query': {},
288 'relative': True 310 'relative': True
311 },
312 "https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip": {
313 'uri': 'https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip',
314 'scheme': 'https',
315 'hostname': 'www.innodisk.com',
316 'port': None,
317 'hostport': 'www.innodisk.com',
318 'path': '/Download_file',
319 'userinfo': '',
320 'userinfo': '',
321 'username': '',
322 'password': '',
323 'params': {"downloadfilename" : "EGPL-T101.zip"},
324 'query': {"9BE0BF6657": None},
325 'relative': False
289 } 326 }
290 327
291 } 328 }
@@ -376,7 +413,7 @@ class FetcherTest(unittest.TestCase):
376 def setUp(self): 413 def setUp(self):
377 self.origdir = os.getcwd() 414 self.origdir = os.getcwd()
378 self.d = bb.data.init() 415 self.d = bb.data.init()
379 self.tempdir = tempfile.mkdtemp() 416 self.tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-")
380 self.dldir = os.path.join(self.tempdir, "download") 417 self.dldir = os.path.join(self.tempdir, "download")
381 os.mkdir(self.dldir) 418 os.mkdir(self.dldir)
382 self.d.setVar("DL_DIR", self.dldir) 419 self.d.setVar("DL_DIR", self.dldir)
@@ -390,57 +427,94 @@ class FetcherTest(unittest.TestCase):
390 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 427 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
391 print("Not cleaning up %s. Please remove manually." % self.tempdir) 428 print("Not cleaning up %s. Please remove manually." % self.tempdir)
392 else: 429 else:
430 bb.process.run('chmod u+rw -R %s' % self.tempdir)
393 bb.utils.prunedir(self.tempdir) 431 bb.utils.prunedir(self.tempdir)
394 432
433 def git(self, cmd, cwd=None):
434 if isinstance(cmd, str):
435 cmd = 'git -c safe.bareRepository=all ' + cmd
436 else:
437 cmd = ['git', '-c', 'safe.bareRepository=all'] + cmd
438 if cwd is None:
439 cwd = self.gitdir
440 return bb.process.run(cmd, cwd=cwd)[0]
441
442 def git_init(self, cwd=None):
443 self.git('init', cwd=cwd)
444 # Explicitly set initial branch to master as
445 # a common setup is to use other default
446 # branch than master.
447 self.git(['checkout', '-b', 'master'], cwd=cwd)
448
449 try:
450 self.git(['config', 'user.email'], cwd=cwd)
451 except bb.process.ExecutionError:
452 self.git(['config', 'user.email', 'you@example.com'], cwd=cwd)
453
454 try:
455 self.git(['config', 'user.name'], cwd=cwd)
456 except bb.process.ExecutionError:
457 self.git(['config', 'user.name', 'Your Name'], cwd=cwd)
458
395class MirrorUriTest(FetcherTest): 459class MirrorUriTest(FetcherTest):
396 460
397 replaceuris = { 461 replaceuris = {
398 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") 462 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/")
399 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz", 463 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
400 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 464 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
401 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 465 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
402 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 466 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
403 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 467 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
404 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") 468 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http")
405 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 469 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
406 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake") 470 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake")
407 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890", 471 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890",
408 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") 472 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache")
409 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 473 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
410 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") 474 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/")
411 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 475 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
412 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") 476 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3")
413 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 477 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
414 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") 478 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz")
415 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 479 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
416 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist") 480 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist")
417 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", 481 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2",
418 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/") 482 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/")
419 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2", 483 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2",
420 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 484 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
421 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 485 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
422 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 486 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
423 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 487 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
424 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") 488 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http")
425 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 489 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
426 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org") 490 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org")
427 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 491 : "http://somewhere2.org/somefile_1.2.3.tar.gz",
428 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/") 492 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/")
429 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 493 : "http://somewhere2.org/somefile_1.2.3.tar.gz",
430 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http") 494 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http")
431 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 495 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
496 ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http")
497 : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
498 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=git;branch=master", "git://someserver.org/bitbake", "git://someotherserver.org/bitbake;protocol=https")
499 : "git://someotherserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=https;branch=master",
500 ("gitsm://git.qemu.org/git/seabios.git/;protocol=https;name=roms/seabios;subpath=roms/seabios;bareclone=1;nobranch=1;rev=1234567890123456789012345678901234567890", "gitsm://.*/.*", "http://petalinux.xilinx.com/sswreleases/rel-v${XILINX_VER_MAIN}/downloads") : "http://petalinux.xilinx.com/sswreleases/rel-v%24%7BXILINX_VER_MAIN%7D/downloads/git2_git.qemu.org.git.seabios.git..tar.gz",
501 ("https://somewhere.org/example/1.0.0/example;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/PATH")
502 : "file:///mirror/example/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz",
503 ("https://somewhere.org/example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/some-example-1.0.0.tgz")
504 : "file:///mirror/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz",
432 505
433 #Renaming files doesn't work 506 #Renaming files doesn't work
434 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz" 507 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
435 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 508 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
436 } 509 }
437 510
438 mirrorvar = "http://.*/.* file:///somepath/downloads/ \n" \ 511 mirrorvar = "http://.*/.* file:///somepath/downloads/ " \
439 "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n" \ 512 "git://someserver.org/bitbake git://git.openembedded.org/bitbake " \
440 "https://.*/.* file:///someotherpath/downloads/ \n" \ 513 "https://.*/.* file:///someotherpath/downloads/ " \
441 "http://.*/.* file:///someotherpath/downloads/ \n" 514 "http://.*/.* file:///someotherpath/downloads/"
442 515
443 def test_urireplace(self): 516 def test_urireplace(self):
517 self.d.setVar("FILESPATH", ".")
444 for k, v in self.replaceuris.items(): 518 for k, v in self.replaceuris.items():
445 ud = bb.fetch.FetchData(k[0], self.d) 519 ud = bb.fetch.FetchData(k[0], self.d)
446 ud.setup_localpath(self.d) 520 ud.setup_localpath(self.d)
@@ -463,8 +537,8 @@ class MirrorUriTest(FetcherTest):
463 537
464 def test_mirror_of_mirror(self): 538 def test_mirror_of_mirror(self):
465 # Test if mirror of a mirror works 539 # Test if mirror of a mirror works
466 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/ \n" 540 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/"
467 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/ \n" 541 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/"
468 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 542 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
469 mirrors = bb.fetch2.mirror_from_string(mirrorvar) 543 mirrors = bb.fetch2.mirror_from_string(mirrorvar)
470 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 544 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
@@ -473,8 +547,8 @@ class MirrorUriTest(FetcherTest):
473 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', 547 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz',
474 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) 548 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz'])
475 549
476 recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ \n" \ 550 recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ " \
477 "https://.*/[^/]* https://BBBB/B/B/B/ \n" 551 "https://.*/[^/]* https://BBBB/B/B/B/"
478 552
479 def test_recursive(self): 553 def test_recursive(self):
480 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 554 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
@@ -488,15 +562,15 @@ class MirrorUriTest(FetcherTest):
488class GitDownloadDirectoryNamingTest(FetcherTest): 562class GitDownloadDirectoryNamingTest(FetcherTest):
489 def setUp(self): 563 def setUp(self):
490 super(GitDownloadDirectoryNamingTest, self).setUp() 564 super(GitDownloadDirectoryNamingTest, self).setUp()
491 self.recipe_url = "git://git.openembedded.org/bitbake" 565 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
492 self.recipe_dir = "git.openembedded.org.bitbake" 566 self.recipe_dir = "git.openembedded.org.bitbake"
493 self.mirror_url = "git://github.com/openembedded/bitbake.git" 567 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
494 self.mirror_dir = "github.com.openembedded.bitbake.git" 568 self.mirror_dir = "github.com.openembedded.bitbake.git"
495 569
496 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 570 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
497 571
498 def setup_mirror_rewrite(self): 572 def setup_mirror_rewrite(self):
499 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n") 573 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url)
500 574
501 @skipIfNoNetwork() 575 @skipIfNoNetwork()
502 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self): 576 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self):
@@ -536,16 +610,16 @@ class GitDownloadDirectoryNamingTest(FetcherTest):
536class TarballNamingTest(FetcherTest): 610class TarballNamingTest(FetcherTest):
537 def setUp(self): 611 def setUp(self):
538 super(TarballNamingTest, self).setUp() 612 super(TarballNamingTest, self).setUp()
539 self.recipe_url = "git://git.openembedded.org/bitbake" 613 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
540 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 614 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
541 self.mirror_url = "git://github.com/openembedded/bitbake.git" 615 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
542 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz" 616 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz"
543 617
544 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 618 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
545 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 619 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
546 620
547 def setup_mirror_rewrite(self): 621 def setup_mirror_rewrite(self):
548 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n") 622 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url)
549 623
550 @skipIfNoNetwork() 624 @skipIfNoNetwork()
551 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self): 625 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self):
@@ -570,9 +644,9 @@ class TarballNamingTest(FetcherTest):
570class GitShallowTarballNamingTest(FetcherTest): 644class GitShallowTarballNamingTest(FetcherTest):
571 def setUp(self): 645 def setUp(self):
572 super(GitShallowTarballNamingTest, self).setUp() 646 super(GitShallowTarballNamingTest, self).setUp()
573 self.recipe_url = "git://git.openembedded.org/bitbake" 647 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
574 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz" 648 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz"
575 self.mirror_url = "git://github.com/openembedded/bitbake.git" 649 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
576 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz" 650 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz"
577 651
578 self.d.setVar('BB_GIT_SHALLOW', '1') 652 self.d.setVar('BB_GIT_SHALLOW', '1')
@@ -580,7 +654,7 @@ class GitShallowTarballNamingTest(FetcherTest):
580 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 654 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
581 655
582 def setup_mirror_rewrite(self): 656 def setup_mirror_rewrite(self):
583 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n") 657 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url)
584 658
585 @skipIfNoNetwork() 659 @skipIfNoNetwork()
586 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self): 660 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self):
@@ -602,6 +676,39 @@ class GitShallowTarballNamingTest(FetcherTest):
602 self.assertIn(self.mirror_tarball, dir) 676 self.assertIn(self.mirror_tarball, dir)
603 677
604 678
679class CleanTarballTest(FetcherTest):
680 def setUp(self):
681 super(CleanTarballTest, self).setUp()
682 self.recipe_url = "git://git.openembedded.org/bitbake;protocol=https"
683 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
684
685 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
686 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
687
688 @skipIfNoNetwork()
689 def test_that_the_tarball_contents_does_not_leak_info(self):
690 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
691
692 fetcher.download()
693
694 fetcher.unpack(self.unpackdir)
695 mtime = bb.process.run('git log --all -1 --format=%ct',
696 cwd=os.path.join(self.unpackdir, 'git'))
697 self.assertEqual(len(mtime), 2)
698 mtime = int(mtime[0])
699
700 archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball))
701 self.assertNotEqual(len(archive.members), 0)
702 for member in archive.members:
703 if member.name == ".":
704 continue
705 self.assertEqual(member.uname, 'oe', "user name for %s differs" % member.name)
706 self.assertEqual(member.uid, 0, "uid for %s differs" % member.name)
707 self.assertEqual(member.gname, 'oe', "group name for %s differs" % member.name)
708 self.assertEqual(member.gid, 0, "gid for %s differs" % member.name)
709 self.assertEqual(member.mtime, mtime, "mtime for %s differs" % member.name)
710
711
605class FetcherLocalTest(FetcherTest): 712class FetcherLocalTest(FetcherTest):
606 def setUp(self): 713 def setUp(self):
607 def touch(fn): 714 def touch(fn):
@@ -619,6 +726,9 @@ class FetcherLocalTest(FetcherTest):
619 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir')) 726 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir'))
620 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e')) 727 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e'))
621 touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device')) 728 touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device'))
729 bb.process.run('tar cf archive.tar -C dir .', cwd=self.localsrcdir)
730 bb.process.run('tar czf archive.tar.gz -C dir .', cwd=self.localsrcdir)
731 bb.process.run('tar cjf archive.tar.bz2 -C dir .', cwd=self.localsrcdir)
622 self.d.setVar("FILESPATH", self.localsrcdir) 732 self.d.setVar("FILESPATH", self.localsrcdir)
623 733
624 def fetchUnpack(self, uris): 734 def fetchUnpack(self, uris):
@@ -632,6 +742,11 @@ class FetcherLocalTest(FetcherTest):
632 flst.sort() 742 flst.sort()
633 return flst 743 return flst
634 744
745 def test_local_checksum_fails_no_file(self):
746 self.d.setVar("SRC_URI", "file://404")
747 with self.assertRaises(bb.BBHandledException):
748 bb.fetch.get_checksum_file_list(self.d)
749
635 def test_local(self): 750 def test_local(self):
636 tree = self.fetchUnpack(['file://a', 'file://dir/c']) 751 tree = self.fetchUnpack(['file://a', 'file://dir/c'])
637 self.assertEqual(tree, ['a', 'dir/c']) 752 self.assertEqual(tree, ['a', 'dir/c'])
@@ -673,57 +788,58 @@ class FetcherLocalTest(FetcherTest):
673 with self.assertRaises(bb.fetch2.UnpackError): 788 with self.assertRaises(bb.fetch2.UnpackError):
674 self.fetchUnpack(['file://a;subdir=/bin/sh']) 789 self.fetchUnpack(['file://a;subdir=/bin/sh'])
675 790
676 def test_local_gitfetch_usehead(self): 791 def test_local_striplevel(self):
792 tree = self.fetchUnpack(['file://archive.tar;subdir=bar;striplevel=1'])
793 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e'])
794
795 def test_local_striplevel_gzip(self):
796 tree = self.fetchUnpack(['file://archive.tar.gz;subdir=bar;striplevel=1'])
797 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e'])
798
799 def test_local_striplevel_bzip2(self):
800 tree = self.fetchUnpack(['file://archive.tar.bz2;subdir=bar;striplevel=1'])
801 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e'])
802
803 def dummyGitTest(self, suffix):
677 # Create dummy local Git repo 804 # Create dummy local Git repo
678 src_dir = tempfile.mkdtemp(dir=self.tempdir, 805 src_dir = tempfile.mkdtemp(dir=self.tempdir,
679 prefix='gitfetch_localusehead_') 806 prefix='gitfetch_localusehead_')
680 src_dir = os.path.abspath(src_dir) 807 self.gitdir = os.path.abspath(src_dir)
681 bb.process.run("git init", cwd=src_dir) 808 self.git_init()
682 bb.process.run("git commit --allow-empty -m'Dummy commit'", 809 self.git(['commit', '--allow-empty', '-m', 'Dummy commit'])
683 cwd=src_dir)
684 # Use other branch than master 810 # Use other branch than master
685 bb.process.run("git checkout -b my-devel", cwd=src_dir) 811 self.git(['checkout', '-b', 'my-devel'])
686 bb.process.run("git commit --allow-empty -m'Dummy commit 2'", 812 self.git(['commit', '--allow-empty', '-m', 'Dummy commit 2'])
687 cwd=src_dir) 813 orig_rev = self.git(['rev-parse', 'HEAD']).strip()
688 stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
689 orig_rev = stdout[0].strip()
690 814
691 # Fetch and check revision 815 # Fetch and check revision
692 self.d.setVar("SRCREV", "AUTOINC") 816 self.d.setVar("SRCREV", "AUTOINC")
693 url = "git://" + src_dir + ";protocol=file;usehead=1" 817 self.d.setVar("__BBSRCREV_SEEN", "1")
818 url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix
694 fetcher = bb.fetch.Fetch([url], self.d) 819 fetcher = bb.fetch.Fetch([url], self.d)
695 fetcher.download() 820 fetcher.download()
696 fetcher.unpack(self.unpackdir) 821 fetcher.unpack(self.unpackdir)
697 stdout = bb.process.run("git rev-parse HEAD", 822 unpack_rev = self.git(['rev-parse', 'HEAD'],
698 cwd=os.path.join(self.unpackdir, 'git')) 823 cwd=os.path.join(self.unpackdir, 'git')).strip()
699 unpack_rev = stdout[0].strip()
700 self.assertEqual(orig_rev, unpack_rev) 824 self.assertEqual(orig_rev, unpack_rev)
701 825
826 def test_local_gitfetch_usehead(self):
827 self.dummyGitTest("usehead=1")
828
702 def test_local_gitfetch_usehead_withname(self): 829 def test_local_gitfetch_usehead_withname(self):
703 # Create dummy local Git repo 830 self.dummyGitTest("usehead=1;name=newName")
704 src_dir = tempfile.mkdtemp(dir=self.tempdir,
705 prefix='gitfetch_localusehead_')
706 src_dir = os.path.abspath(src_dir)
707 bb.process.run("git init", cwd=src_dir)
708 bb.process.run("git commit --allow-empty -m'Dummy commit'",
709 cwd=src_dir)
710 # Use other branch than master
711 bb.process.run("git checkout -b my-devel", cwd=src_dir)
712 bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
713 cwd=src_dir)
714 stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
715 orig_rev = stdout[0].strip()
716 831
717 # Fetch and check revision 832 def test_local_gitfetch_shared(self):
718 self.d.setVar("SRCREV", "AUTOINC") 833 self.dummyGitTest("usehead=1;name=sharedName")
719 url = "git://" + src_dir + ";protocol=file;usehead=1;name=newName" 834 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
720 fetcher = bb.fetch.Fetch([url], self.d) 835 self.assertTrue(os.path.exists(alt))
721 fetcher.download() 836
722 fetcher.unpack(self.unpackdir) 837 def test_local_gitfetch_noshared(self):
723 stdout = bb.process.run("git rev-parse HEAD", 838 self.d.setVar('BB_GIT_NOSHARED', '1')
724 cwd=os.path.join(self.unpackdir, 'git')) 839 self.unpackdir += '_noshared'
725 unpack_rev = stdout[0].strip() 840 self.dummyGitTest("usehead=1;name=noSharedName")
726 self.assertEqual(orig_rev, unpack_rev) 841 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
842 self.assertFalse(os.path.exists(alt))
727 843
728class FetcherNoNetworkTest(FetcherTest): 844class FetcherNoNetworkTest(FetcherTest):
729 def setUp(self): 845 def setUp(self):
@@ -831,12 +947,12 @@ class FetcherNoNetworkTest(FetcherTest):
831class FetcherNetworkTest(FetcherTest): 947class FetcherNetworkTest(FetcherTest):
832 @skipIfNoNetwork() 948 @skipIfNoNetwork()
833 def test_fetch(self): 949 def test_fetch(self):
834 fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 950 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
835 fetcher.download() 951 fetcher.download()
836 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 952 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
837 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892) 953 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892)
838 self.d.setVar("BB_NO_NETWORK", "1") 954 self.d.setVar("BB_NO_NETWORK", "1")
839 fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 955 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
840 fetcher.download() 956 fetcher.download()
841 fetcher.unpack(self.unpackdir) 957 fetcher.unpack(self.unpackdir)
842 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9) 958 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9)
@@ -844,21 +960,22 @@ class FetcherNetworkTest(FetcherTest):
844 960
845 @skipIfNoNetwork() 961 @skipIfNoNetwork()
846 def test_fetch_mirror(self): 962 def test_fetch_mirror(self):
847 self.d.setVar("MIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake") 963 self.d.setVar("MIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
848 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 964 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
849 fetcher.download() 965 fetcher.download()
850 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 966 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
851 967
852 @skipIfNoNetwork() 968 @skipIfNoNetwork()
853 def test_fetch_mirror_of_mirror(self): 969 def test_fetch_mirror_of_mirror(self):
854 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ \n http://invalid2.yoctoproject.org/.* http://downloads.yoctoproject.org/releases/bitbake") 970 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake")
855 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 971 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
856 fetcher.download() 972 fetcher.download()
857 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 973 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
858 974
859 @skipIfNoNetwork() 975 @skipIfNoNetwork()
860 def test_fetch_file_mirror_of_mirror(self): 976 def test_fetch_file_mirror_of_mirror(self):
861 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ \n file:///some1where/.* file://some2where/ \n file://some2where/.* http://downloads.yoctoproject.org/releases/bitbake") 977 self.d.setVar("FILESPATH", ".")
978 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake")
862 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 979 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
863 os.mkdir(self.dldir + "/some2where") 980 os.mkdir(self.dldir + "/some2where")
864 fetcher.download() 981 fetcher.download()
@@ -866,16 +983,46 @@ class FetcherNetworkTest(FetcherTest):
866 983
867 @skipIfNoNetwork() 984 @skipIfNoNetwork()
868 def test_fetch_premirror(self): 985 def test_fetch_premirror(self):
869 self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake") 986 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
870 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 987 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
871 fetcher.download() 988 fetcher.download()
872 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 989 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
873 990
874 @skipIfNoNetwork() 991 @skipIfNoNetwork()
992 def test_fetch_specify_downloadfilename(self):
993 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d)
994 fetcher.download()
995 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749)
996
997 @skipIfNoNetwork()
998 def test_fetch_premirror_specify_downloadfilename_regex_uri(self):
999 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/")
1000 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d)
1001 fetcher.download()
1002 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
1003
1004 @skipIfNoNetwork()
1005 # BZ13039
1006 def test_fetch_premirror_specify_downloadfilename_specific_uri(self):
1007 self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake")
1008 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d)
1009 fetcher.download()
1010 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
1011
1012 @skipIfNoNetwork()
1013 def test_fetch_premirror_use_downloadfilename_to_fetch(self):
1014 # Ensure downloadfilename is used when fetching from premirror.
1015 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
1016 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d)
1017 fetcher.download()
1018 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
1019
1020 @skipIfNoNetwork()
875 def gitfetcher(self, url1, url2): 1021 def gitfetcher(self, url1, url2):
876 def checkrevision(self, fetcher): 1022 def checkrevision(self, fetcher):
877 fetcher.unpack(self.unpackdir) 1023 fetcher.unpack(self.unpackdir)
878 revision = bb.process.run("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git")[0].strip() 1024 revision = self.git(['rev-parse', 'HEAD'],
1025 cwd=os.path.join(self.unpackdir, 'git')).strip()
879 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 1026 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
880 1027
881 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 1028 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
@@ -893,25 +1040,25 @@ class FetcherNetworkTest(FetcherTest):
893 1040
894 @skipIfNoNetwork() 1041 @skipIfNoNetwork()
895 def test_gitfetch(self): 1042 def test_gitfetch(self):
896 url1 = url2 = "git://git.openembedded.org/bitbake" 1043 url1 = url2 = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
897 self.gitfetcher(url1, url2) 1044 self.gitfetcher(url1, url2)
898 1045
899 @skipIfNoNetwork() 1046 @skipIfNoNetwork()
900 def test_gitfetch_goodsrcrev(self): 1047 def test_gitfetch_goodsrcrev(self):
901 # SRCREV is set but matches rev= parameter 1048 # SRCREV is set but matches rev= parameter
902 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5" 1049 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https"
903 self.gitfetcher(url1, url2) 1050 self.gitfetcher(url1, url2)
904 1051
905 @skipIfNoNetwork() 1052 @skipIfNoNetwork()
906 def test_gitfetch_badsrcrev(self): 1053 def test_gitfetch_badsrcrev(self):
907 # SRCREV is set but does not match rev= parameter 1054 # SRCREV is set but does not match rev= parameter
908 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5" 1055 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https"
909 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1056 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
910 1057
911 @skipIfNoNetwork() 1058 @skipIfNoNetwork()
912 def test_gitfetch_tagandrev(self): 1059 def test_gitfetch_tagandrev(self):
913 # SRCREV is set but does not match rev= parameter 1060 # SRCREV is set but does not match rev= parameter
914 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5" 1061 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5;protocol=https"
915 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1062 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
916 1063
917 @skipIfNoNetwork() 1064 @skipIfNoNetwork()
@@ -920,7 +1067,7 @@ class FetcherNetworkTest(FetcherTest):
920 # `usehead=1' and instead fetch the specified SRCREV. See 1067 # `usehead=1' and instead fetch the specified SRCREV. See
921 # test_local_gitfetch_usehead() for a positive use of the usehead 1068 # test_local_gitfetch_usehead() for a positive use of the usehead
922 # feature. 1069 # feature.
923 url = "git://git.openembedded.org/bitbake;usehead=1" 1070 url = "git://git.openembedded.org/bitbake;usehead=1;branch=master;protocol=https"
924 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1071 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
925 1072
926 @skipIfNoNetwork() 1073 @skipIfNoNetwork()
@@ -929,38 +1076,38 @@ class FetcherNetworkTest(FetcherTest):
929 # `usehead=1' and instead fetch the specified SRCREV. See 1076 # `usehead=1' and instead fetch the specified SRCREV. See
930 # test_local_gitfetch_usehead() for a positive use of the usehead 1077 # test_local_gitfetch_usehead() for a positive use of the usehead
931 # feature. 1078 # feature.
932 url = "git://git.openembedded.org/bitbake;usehead=1;name=newName" 1079 url = "git://git.openembedded.org/bitbake;usehead=1;name=newName;branch=master;protocol=https"
933 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1080 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
934 1081
935 @skipIfNoNetwork() 1082 @skipIfNoNetwork()
936 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self): 1083 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self):
937 recipeurl = "git://git.openembedded.org/bitbake" 1084 recipeurl = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
938 mirrorurl = "git://someserver.org/bitbake" 1085 mirrorurl = "git://someserver.org/bitbake;branch=master;protocol=https"
939 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n") 1086 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake")
940 self.gitfetcher(recipeurl, mirrorurl) 1087 self.gitfetcher(recipeurl, mirrorurl)
941 1088
942 @skipIfNoNetwork() 1089 @skipIfNoNetwork()
943 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self): 1090 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self):
944 recipeurl = "git://someserver.org/bitbake" 1091 recipeurl = "git://someserver.org/bitbake;branch=master;protocol=https"
945 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n") 1092 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake")
946 self.gitfetcher(recipeurl, recipeurl) 1093 self.gitfetcher(recipeurl, recipeurl)
947 1094
948 @skipIfNoNetwork() 1095 @skipIfNoNetwork()
949 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self): 1096 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self):
950 realurl = "git://git.openembedded.org/bitbake" 1097 realurl = "https://git.openembedded.org/bitbake"
951 recipeurl = "git://someserver.org/bitbake" 1098 recipeurl = "git://someserver.org/bitbake;protocol=https"
952 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git") 1099 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git")
953 os.chdir(self.tempdir) 1100 os.chdir(self.tempdir)
954 bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True) 1101 self.git(['clone', realurl, self.sourcedir], cwd=self.tempdir)
955 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (recipeurl, self.sourcedir)) 1102 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipeurl, self.sourcedir))
956 self.gitfetcher(recipeurl, recipeurl) 1103 self.gitfetcher(recipeurl, recipeurl)
957 1104
958 @skipIfNoNetwork() 1105 @skipIfNoNetwork()
959 def test_git_submodule(self): 1106 def test_git_submodule(self):
960 # URL with ssh submodules 1107 # URL with ssh submodules
961 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7" 1108 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master;protocol=https"
962 # Original URL (comment this if you have ssh access to git.yoctoproject.org) 1109 # Original URL (comment this if you have ssh access to git.yoctoproject.org)
963 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee" 1110 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master;protocol=https"
964 fetcher = bb.fetch.Fetch([url], self.d) 1111 fetcher = bb.fetch.Fetch([url], self.d)
965 fetcher.download() 1112 fetcher.download()
966 # Previous cwd has been deleted 1113 # Previous cwd has been deleted
@@ -977,10 +1124,29 @@ class FetcherNetworkTest(FetcherTest):
977 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing') 1124 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
978 1125
979 @skipIfNoNetwork() 1126 @skipIfNoNetwork()
1127 def test_git_submodule_restricted_network_premirrors(self):
1128 # this test is to ensure that premirrors will be tried in restricted network
1129 # that is, BB_ALLOWED_NETWORKS does not contain the domain the url uses
1130 url = "gitsm://github.com/grpc/grpc.git;protocol=https;name=grpc;branch=v1.60.x;rev=0ef13a7555dbaadd4633399242524129eef5e231"
1131 # create a download directory to be used as premirror later
1132 tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-")
1133 dl_premirror = os.path.join(tempdir, "download-premirror")
1134 os.mkdir(dl_premirror)
1135 self.d.setVar("DL_DIR", dl_premirror)
1136 fetcher = bb.fetch.Fetch([url], self.d)
1137 fetcher.download()
1138 # now use the premirror in restricted network
1139 self.d.setVar("DL_DIR", self.dldir)
1140 self.d.setVar("PREMIRRORS", "gitsm://.*/.* gitsm://%s/git2/MIRRORNAME;protocol=file" % dl_premirror)
1141 self.d.setVar("BB_ALLOWED_NETWORKS", "*.some.domain")
1142 fetcher = bb.fetch.Fetch([url], self.d)
1143 fetcher.download()
1144
1145 @skipIfNoNetwork()
980 def test_git_submodule_dbus_broker(self): 1146 def test_git_submodule_dbus_broker(self):
981 # The following external repositories have show failures in fetch and unpack operations 1147 # The following external repositories have show failures in fetch and unpack operations
982 # We want to avoid regressions! 1148 # We want to avoid regressions!
983 url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main" 1149 url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main"
984 fetcher = bb.fetch.Fetch([url], self.d) 1150 fetcher = bb.fetch.Fetch([url], self.d)
985 fetcher.download() 1151 fetcher.download()
986 # Previous cwd has been deleted 1152 # Previous cwd has been deleted
@@ -996,7 +1162,7 @@ class FetcherNetworkTest(FetcherTest):
996 1162
997 @skipIfNoNetwork() 1163 @skipIfNoNetwork()
998 def test_git_submodule_CLI11(self): 1164 def test_git_submodule_CLI11(self):
999 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf" 1165 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main"
1000 fetcher = bb.fetch.Fetch([url], self.d) 1166 fetcher = bb.fetch.Fetch([url], self.d)
1001 fetcher.download() 1167 fetcher.download()
1002 # Previous cwd has been deleted 1168 # Previous cwd has been deleted
@@ -1011,12 +1177,12 @@ class FetcherNetworkTest(FetcherTest):
1011 @skipIfNoNetwork() 1177 @skipIfNoNetwork()
1012 def test_git_submodule_update_CLI11(self): 1178 def test_git_submodule_update_CLI11(self):
1013 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """ 1179 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
1014 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714" 1180 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main"
1015 fetcher = bb.fetch.Fetch([url], self.d) 1181 fetcher = bb.fetch.Fetch([url], self.d)
1016 fetcher.download() 1182 fetcher.download()
1017 1183
1018 # CLI11 that pulls in a newer nlohmann-json 1184 # CLI11 that pulls in a newer nlohmann-json
1019 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca" 1185 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main"
1020 fetcher = bb.fetch.Fetch([url], self.d) 1186 fetcher = bb.fetch.Fetch([url], self.d)
1021 fetcher.download() 1187 fetcher.download()
1022 # Previous cwd has been deleted 1188 # Previous cwd has been deleted
@@ -1030,7 +1196,7 @@ class FetcherNetworkTest(FetcherTest):
1030 1196
1031 @skipIfNoNetwork() 1197 @skipIfNoNetwork()
1032 def test_git_submodule_aktualizr(self): 1198 def test_git_submodule_aktualizr(self):
1033 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" 1199 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
1034 fetcher = bb.fetch.Fetch([url], self.d) 1200 fetcher = bb.fetch.Fetch([url], self.d)
1035 fetcher.download() 1201 fetcher.download()
1036 # Previous cwd has been deleted 1202 # Previous cwd has been deleted
@@ -1050,7 +1216,7 @@ class FetcherNetworkTest(FetcherTest):
1050 """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """ 1216 """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """
1051 1217
1052 # This repository also has submodules where the module (name), path and url do not align 1218 # This repository also has submodules where the module (name), path and url do not align
1053 url = "gitsm://github.com/azure/iotedge.git;protocol=git;rev=d76e0316c6f324345d77c48a83ce836d09392699" 1219 url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699;branch=main"
1054 fetcher = bb.fetch.Fetch([url], self.d) 1220 fetcher = bb.fetch.Fetch([url], self.d)
1055 fetcher.download() 1221 fetcher.download()
1056 # Previous cwd has been deleted 1222 # Previous cwd has been deleted
@@ -1073,6 +1239,15 @@ class FetcherNetworkTest(FetcherTest):
1073 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1239 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
1074 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1240 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
1075 1241
1242 @skipIfNoNetwork()
1243 def test_git_submodule_reference_to_parent(self):
1244 self.recipe_url = "gitsm://github.com/gflags/gflags.git;protocol=https;branch=master"
1245 self.d.setVar("SRCREV", "14e1138441bbbb584160cb1c0a0426ec1bac35f1")
1246 with Timeout(60):
1247 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
1248 with self.assertRaises(bb.fetch2.FetchError):
1249 fetcher.download()
1250
1076class SVNTest(FetcherTest): 1251class SVNTest(FetcherTest):
1077 def skipIfNoSvn(): 1252 def skipIfNoSvn():
1078 import shutil 1253 import shutil
@@ -1107,8 +1282,9 @@ class SVNTest(FetcherTest):
1107 cwd=repo_dir) 1282 cwd=repo_dir)
1108 1283
1109 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir) 1284 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
1110 # Github will emulate SVN. Use this to check if we're downloding... 1285 # Github won't emulate SVN anymore (see https://github.blog/2023-01-20-sunsetting-subversion-support/)
1111 bb.process.run("svn propset svn:externals 'bitbake svn://vcs.pcre.org/pcre2/code' .", 1286 # Use still accessible svn repo (only trunk to avoid longer downloads)
1287 bb.process.run("svn propset svn:externals 'bitbake https://svn.apache.org/repos/asf/serf/trunk' .",
1112 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1288 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
1113 bb.process.run("svn commit --non-interactive -m 'Add external'", 1289 bb.process.run("svn commit --non-interactive -m 'Add external'",
1114 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1290 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
@@ -1136,8 +1312,8 @@ class SVNTest(FetcherTest):
1136 1312
1137 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1313 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
1138 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1314 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
1139 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should NOT exist") 1315 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should NOT exist")
1140 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should NOT exit") 1316 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should NOT exit")
1141 1317
1142 @skipIfNoSvn() 1318 @skipIfNoSvn()
1143 def test_external_svn(self): 1319 def test_external_svn(self):
@@ -1150,49 +1326,49 @@ class SVNTest(FetcherTest):
1150 1326
1151 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1327 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
1152 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1328 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
1153 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should exist") 1329 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should exist")
1154 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should exit") 1330 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should exit")
1155 1331
1156class TrustedNetworksTest(FetcherTest): 1332class TrustedNetworksTest(FetcherTest):
1157 def test_trusted_network(self): 1333 def test_trusted_network(self):
1158 # Ensure trusted_network returns False when the host IS in the list. 1334 # Ensure trusted_network returns False when the host IS in the list.
1159 url = "git://Someserver.org/foo;rev=1" 1335 url = "git://Someserver.org/foo;rev=1;branch=master"
1160 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org") 1336 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org")
1161 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1337 self.assertTrue(bb.fetch.trusted_network(self.d, url))
1162 1338
1163 def test_wild_trusted_network(self): 1339 def test_wild_trusted_network(self):
1164 # Ensure trusted_network returns true when the *.host IS in the list. 1340 # Ensure trusted_network returns true when the *.host IS in the list.
1165 url = "git://Someserver.org/foo;rev=1" 1341 url = "git://Someserver.org/foo;rev=1;branch=master"
1166 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1342 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1167 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1343 self.assertTrue(bb.fetch.trusted_network(self.d, url))
1168 1344
1169 def test_prefix_wild_trusted_network(self): 1345 def test_prefix_wild_trusted_network(self):
1170 # Ensure trusted_network returns true when the prefix matches *.host. 1346 # Ensure trusted_network returns true when the prefix matches *.host.
1171 url = "git://git.Someserver.org/foo;rev=1" 1347 url = "git://git.Someserver.org/foo;rev=1;branch=master"
1172 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1348 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1173 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1349 self.assertTrue(bb.fetch.trusted_network(self.d, url))
1174 1350
1175 def test_two_prefix_wild_trusted_network(self): 1351 def test_two_prefix_wild_trusted_network(self):
1176 # Ensure trusted_network returns true when the prefix matches *.host. 1352 # Ensure trusted_network returns true when the prefix matches *.host.
1177 url = "git://something.git.Someserver.org/foo;rev=1" 1353 url = "git://something.git.Someserver.org/foo;rev=1;branch=master"
1178 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1354 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1179 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1355 self.assertTrue(bb.fetch.trusted_network(self.d, url))
1180 1356
1181 def test_port_trusted_network(self): 1357 def test_port_trusted_network(self):
1182 # Ensure trusted_network returns True, even if the url specifies a port. 1358 # Ensure trusted_network returns True, even if the url specifies a port.
1183 url = "git://someserver.org:8080/foo;rev=1" 1359 url = "git://someserver.org:8080/foo;rev=1;branch=master"
1184 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org") 1360 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org")
1185 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1361 self.assertTrue(bb.fetch.trusted_network(self.d, url))
1186 1362
1187 def test_untrusted_network(self): 1363 def test_untrusted_network(self):
1188 # Ensure trusted_network returns False when the host is NOT in the list. 1364 # Ensure trusted_network returns False when the host is NOT in the list.
1189 url = "git://someserver.org/foo;rev=1" 1365 url = "git://someserver.org/foo;rev=1;branch=master"
1190 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1366 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
1191 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1367 self.assertFalse(bb.fetch.trusted_network(self.d, url))
1192 1368
1193 def test_wild_untrusted_network(self): 1369 def test_wild_untrusted_network(self):
1194 # Ensure trusted_network returns False when the host is NOT in the list. 1370 # Ensure trusted_network returns False when the host is NOT in the list.
1195 url = "git://*.someserver.org/foo;rev=1" 1371 url = "git://*.someserver.org/foo;rev=1;branch=master"
1196 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1372 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
1197 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1373 self.assertFalse(bb.fetch.trusted_network(self.d, url))
1198 1374
@@ -1202,14 +1378,17 @@ class URLHandle(unittest.TestCase):
1202 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}), 1378 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}),
1203 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}), 1379 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}),
1204 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])), 1380 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])),
1205 "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'}), 1381 "git://git.openembedded.org/bitbake;branch=@foo;protocol=https" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo', 'protocol' : 'https'}),
1206 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}), 1382 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}),
1383 "https://somesite.com/somerepo.git;user=anyUser:idtoken=1234" : ('https', 'somesite.com', '/somerepo.git', '', '', {'user': 'anyUser:idtoken=1234'}),
1384 r'git://s.o-me_ONE:!#$%^&*()-_={}[]\|:?,.<>~`@git.openembedded.org/bitbake;branch=main;protocol=https': ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', r'!#$%^&*()-_={}[]\|:?,.<>~`', {'branch': 'main', 'protocol' : 'https'}),
1207 } 1385 }
1208 # we require a pathname to encodeurl but users can still pass such urls to 1386 # we require a pathname to encodeurl but users can still pass such urls to
1209 # decodeurl and we need to handle them 1387 # decodeurl and we need to handle them
1210 decodedata = datatable.copy() 1388 decodedata = datatable.copy()
1211 decodedata.update({ 1389 decodedata.update({
1212 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}), 1390 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}),
1391 "npmsw://some.registry.url;package=@pkg;version=latest": ('npmsw', 'some.registry.url', '/', '', '', {'package': '@pkg', 'version': 'latest'}),
1213 }) 1392 })
1214 1393
1215 def test_decodeurl(self): 1394 def test_decodeurl(self):
@@ -1226,37 +1405,39 @@ class FetchLatestVersionTest(FetcherTest):
1226 1405
1227 test_git_uris = { 1406 test_git_uris = {
1228 # version pattern "X.Y.Z" 1407 # version pattern "X.Y.Z"
1229 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "") 1408 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "", "")
1230 : "1.99.4", 1409 : "1.99.4",
1231 # version pattern "vX.Y" 1410 # version pattern "vX.Y"
1232 # mirror of git.infradead.org since network issues interfered with testing 1411 # mirror of git.infradead.org since network issues interfered with testing
1233 ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "") 1412 ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master;protocol=https", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "", "")
1234 : "1.5.0", 1413 : "1.5.0",
1235 # version pattern "pkg_name-X.Y" 1414 # version pattern "pkg_name-X.Y"
1236 # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing 1415 # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing
1237 ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "") 1416 ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master;protocol=https", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "", "")
1238 : "1.0", 1417 : "1.0",
1239 # version pattern "pkg_name-vX.Y.Z" 1418 # version pattern "pkg_name-vX.Y.Z"
1240 ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "") 1419 ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master;protocol=https", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "", "")
1241 : "1.4.0", 1420 : "1.4.0",
1242 # combination version pattern 1421 # combination version pattern
1243 ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "") 1422 ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "", "")
1244 : "1.2.0", 1423 : "1.2.0",
1245 ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "") 1424 ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "")
1246 : "2014.01", 1425 : "2014.01",
1247 # version pattern "yyyymmdd" 1426 # version pattern "yyyymmdd"
1248 ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https", "4ed19e11c2975105b71b956440acdb25d46a347d", "") 1427 ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "", "")
1249 : "20120614", 1428 : "20120614",
1250 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX 1429 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
1251 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing 1430 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing
1252 ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))") 1431 ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master;protocol=https", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))", "")
1253 : "0.4.3", 1432 : "0.4.3",
1254 ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", "(?P<pver>(([0-9][\.|_]?)+[0-9]))") 1433 ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master;protocol=https", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))", "")
1255 : "11.0.0", 1434 : "11.0.0",
1256 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", "chkconfig\-(?P<pver>((\d+[\.\-_]*)+))") 1435 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))", "")
1257 : "1.3.59", 1436 : "1.3.59",
1258 ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", "(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))") 1437 ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))", "")
1259 : "3.82+dbg0.9", 1438 : "3.82+dbg0.9",
1439 ("sysdig", "git://github.com/draios/sysdig.git;branch=dev;protocol=https", "4fb6288275f567f63515df0ff0a6518043ecfa9b", r"^(?P<pver>\d+(\.\d+)+)", "10.0.0")
1440 : "0.28.0",
1260 } 1441 }
1261 1442
1262 test_wget_uris = { 1443 test_wget_uris = {
@@ -1272,13 +1453,16 @@ class FetchLatestVersionTest(FetcherTest):
1272 # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz 1453 # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz
1273 ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "") 1454 ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "")
1274 : "2.8.12.1", 1455 : "2.8.12.1",
1456 # https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz
1457 ("libxml2", "/software/libxml2/2.9/libxml2-2.9.14.tar.xz", "", "")
1458 : "2.10.3",
1275 # 1459 #
1276 # packages with versions only in current directory 1460 # packages with versions only in current directory
1277 # 1461 #
1278 # http://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2 1462 # https://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2
1279 ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "") 1463 ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "")
1280 : "2.19", 1464 : "2.19",
1281 # http://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2 1465 # https://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2
1282 ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "") 1466 ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "")
1283 : "20120814", 1467 : "20120814",
1284 # 1468 #
@@ -1295,12 +1479,18 @@ class FetchLatestVersionTest(FetcherTest):
1295 # 1479 #
1296 # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2 1480 # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2
1297 # https://github.com/apple/cups/releases 1481 # https://github.com/apple/cups/releases
1298 ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", "(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz") 1482 ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", r"(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz")
1299 : "2.0.0", 1483 : "2.0.0",
1300 # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz 1484 # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz
1301 # http://ftp.debian.org/debian/pool/main/d/db5.3/ 1485 # http://ftp.debian.org/debian/pool/main/d/db5.3/
1302 ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", "(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz") 1486 ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", r"(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz")
1303 : "5.3.10", 1487 : "5.3.10",
1488 #
1489 # packages where the tarball compression changed in the new version
1490 #
1491 # http://ftp.debian.org/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz
1492 ("minicom", "/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz", "", "")
1493 : "2.8",
1304 } 1494 }
1305 1495
1306 @skipIfNoNetwork() 1496 @skipIfNoNetwork()
@@ -1315,6 +1505,9 @@ class FetchLatestVersionTest(FetcherTest):
1315 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1505 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
1316 r = bb.utils.vercmp_string(v, verstring) 1506 r = bb.utils.vercmp_string(v, verstring)
1317 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1507 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
1508 if k[4]:
1509 r = bb.utils.vercmp_string(verstring, k[4])
1510 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], verstring, k[4]))
1318 1511
1319 def test_wget_latest_versionstring(self): 1512 def test_wget_latest_versionstring(self):
1320 testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata" 1513 testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata"
@@ -1341,17 +1534,14 @@ class FetchLatestVersionTest(FetcherTest):
1341 1534
1342 1535
1343class FetchCheckStatusTest(FetcherTest): 1536class FetchCheckStatusTest(FetcherTest):
1344 test_wget_uris = ["http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz", 1537 test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
1345 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz", 1538 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
1346 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz", 1539 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
1347 "https://yoctoproject.org/", 1540 "https://yoctoproject.org/",
1348 "https://yoctoproject.org/documentation", 1541 "https://docs.yoctoproject.org",
1349 "http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", 1542 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
1350 "http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", 1543 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
1351 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz", 1544 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz",
1352 "http://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz",
1353 "https://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz",
1354 "https://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz",
1355 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD 1545 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD
1356 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz" 1546 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz"
1357 ] 1547 ]
@@ -1389,7 +1579,7 @@ class GitMakeShallowTest(FetcherTest):
1389 FetcherTest.setUp(self) 1579 FetcherTest.setUp(self)
1390 self.gitdir = os.path.join(self.tempdir, 'gitshallow') 1580 self.gitdir = os.path.join(self.tempdir, 'gitshallow')
1391 bb.utils.mkdirhier(self.gitdir) 1581 bb.utils.mkdirhier(self.gitdir)
1392 bb.process.run('git init', cwd=self.gitdir) 1582 self.git_init()
1393 1583
1394 def assertRefs(self, expected_refs): 1584 def assertRefs(self, expected_refs):
1395 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines() 1585 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
@@ -1403,13 +1593,6 @@ class GitMakeShallowTest(FetcherTest):
1403 actual_count = len(revs.splitlines()) 1593 actual_count = len(revs.splitlines())
1404 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1594 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1405 1595
1406 def git(self, cmd):
1407 if isinstance(cmd, str):
1408 cmd = 'git ' + cmd
1409 else:
1410 cmd = ['git'] + cmd
1411 return bb.process.run(cmd, cwd=self.gitdir)[0]
1412
1413 def make_shallow(self, args=None): 1596 def make_shallow(self, args=None):
1414 if args is None: 1597 if args is None:
1415 args = ['HEAD'] 1598 args = ['HEAD']
@@ -1512,13 +1695,13 @@ class GitShallowTest(FetcherTest):
1512 self.srcdir = os.path.join(self.tempdir, 'gitsource') 1695 self.srcdir = os.path.join(self.tempdir, 'gitsource')
1513 1696
1514 bb.utils.mkdirhier(self.srcdir) 1697 bb.utils.mkdirhier(self.srcdir)
1515 self.git('init', cwd=self.srcdir) 1698 self.git_init(cwd=self.srcdir)
1516 self.d.setVar('WORKDIR', self.tempdir) 1699 self.d.setVar('WORKDIR', self.tempdir)
1517 self.d.setVar('S', self.gitdir) 1700 self.d.setVar('S', self.gitdir)
1518 self.d.delVar('PREMIRRORS') 1701 self.d.delVar('PREMIRRORS')
1519 self.d.delVar('MIRRORS') 1702 self.d.delVar('MIRRORS')
1520 1703
1521 uri = 'git://%s;protocol=file;subdir=${S}' % self.srcdir 1704 uri = 'git://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
1522 self.d.setVar('SRC_URI', uri) 1705 self.d.setVar('SRC_URI', uri)
1523 self.d.setVar('SRCREV', '${AUTOREV}') 1706 self.d.setVar('SRCREV', '${AUTOREV}')
1524 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 1707 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
@@ -1526,6 +1709,7 @@ class GitShallowTest(FetcherTest):
1526 self.d.setVar('BB_GIT_SHALLOW', '1') 1709 self.d.setVar('BB_GIT_SHALLOW', '1')
1527 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0') 1710 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
1528 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 1711 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
1712 self.d.setVar("__BBSRCREV_SEEN", "1")
1529 1713
1530 def assertRefs(self, expected_refs, cwd=None): 1714 def assertRefs(self, expected_refs, cwd=None):
1531 if cwd is None: 1715 if cwd is None:
@@ -1543,15 +1727,6 @@ class GitShallowTest(FetcherTest):
1543 actual_count = len(revs.splitlines()) 1727 actual_count = len(revs.splitlines())
1544 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1728 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1545 1729
1546 def git(self, cmd, cwd=None):
1547 if isinstance(cmd, str):
1548 cmd = 'git ' + cmd
1549 else:
1550 cmd = ['git'] + cmd
1551 if cwd is None:
1552 cwd = self.gitdir
1553 return bb.process.run(cmd, cwd=cwd)[0]
1554
1555 def add_empty_file(self, path, cwd=None, msg=None): 1730 def add_empty_file(self, path, cwd=None, msg=None):
1556 if msg is None: 1731 if msg is None:
1557 msg = path 1732 msg = path
@@ -1594,6 +1769,7 @@ class GitShallowTest(FetcherTest):
1594 1769
1595 # fetch and unpack, from the shallow tarball 1770 # fetch and unpack, from the shallow tarball
1596 bb.utils.remove(self.gitdir, recurse=True) 1771 bb.utils.remove(self.gitdir, recurse=True)
1772 bb.process.run('chmod u+w -R "%s"' % ud.clonedir)
1597 bb.utils.remove(ud.clonedir, recurse=True) 1773 bb.utils.remove(ud.clonedir, recurse=True)
1598 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True) 1774 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
1599 1775
@@ -1745,7 +1921,7 @@ class GitShallowTest(FetcherTest):
1745 1921
1746 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1922 smdir = os.path.join(self.tempdir, 'gitsubmodule')
1747 bb.utils.mkdirhier(smdir) 1923 bb.utils.mkdirhier(smdir)
1748 self.git('init', cwd=smdir) 1924 self.git_init(cwd=smdir)
1749 # Make this look like it was cloned from a remote... 1925 # Make this look like it was cloned from a remote...
1750 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1926 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
1751 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1927 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
@@ -1753,11 +1929,11 @@ class GitShallowTest(FetcherTest):
1753 self.add_empty_file('bsub', cwd=smdir) 1929 self.add_empty_file('bsub', cwd=smdir)
1754 1930
1755 self.git('submodule init', cwd=self.srcdir) 1931 self.git('submodule init', cwd=self.srcdir)
1756 self.git('submodule add file://%s' % smdir, cwd=self.srcdir) 1932 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir)
1757 self.git('submodule update', cwd=self.srcdir) 1933 self.git('submodule update', cwd=self.srcdir)
1758 self.git('commit -m submodule -a', cwd=self.srcdir) 1934 self.git('commit -m submodule -a', cwd=self.srcdir)
1759 1935
1760 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir 1936 uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
1761 fetcher, ud = self.fetch_shallow(uri) 1937 fetcher, ud = self.fetch_shallow(uri)
1762 1938
1763 # Verify the main repository is shallow 1939 # Verify the main repository is shallow
@@ -1775,7 +1951,7 @@ class GitShallowTest(FetcherTest):
1775 1951
1776 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1952 smdir = os.path.join(self.tempdir, 'gitsubmodule')
1777 bb.utils.mkdirhier(smdir) 1953 bb.utils.mkdirhier(smdir)
1778 self.git('init', cwd=smdir) 1954 self.git_init(cwd=smdir)
1779 # Make this look like it was cloned from a remote... 1955 # Make this look like it was cloned from a remote...
1780 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1956 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
1781 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1957 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
@@ -1783,7 +1959,7 @@ class GitShallowTest(FetcherTest):
1783 self.add_empty_file('bsub', cwd=smdir) 1959 self.add_empty_file('bsub', cwd=smdir)
1784 1960
1785 self.git('submodule init', cwd=self.srcdir) 1961 self.git('submodule init', cwd=self.srcdir)
1786 self.git('submodule add file://%s' % smdir, cwd=self.srcdir) 1962 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir)
1787 self.git('submodule update', cwd=self.srcdir) 1963 self.git('submodule update', cwd=self.srcdir)
1788 self.git('commit -m submodule -a', cwd=self.srcdir) 1964 self.git('commit -m submodule -a', cwd=self.srcdir)
1789 1965
@@ -1794,8 +1970,8 @@ class GitShallowTest(FetcherTest):
1794 1970
1795 # Set up the mirror 1971 # Set up the mirror
1796 mirrordir = os.path.join(self.tempdir, 'mirror') 1972 mirrordir = os.path.join(self.tempdir, 'mirror')
1797 os.rename(self.dldir, mirrordir) 1973 bb.utils.rename(self.dldir, mirrordir)
1798 self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/\n' % mirrordir) 1974 self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/' % mirrordir)
1799 1975
1800 # Fetch from the mirror 1976 # Fetch from the mirror
1801 bb.utils.remove(self.dldir, recurse=True) 1977 bb.utils.remove(self.dldir, recurse=True)
@@ -1818,10 +1994,10 @@ class GitShallowTest(FetcherTest):
1818 self.git('annex init', cwd=self.srcdir) 1994 self.git('annex init', cwd=self.srcdir)
1819 open(os.path.join(self.srcdir, 'c'), 'w').close() 1995 open(os.path.join(self.srcdir, 'c'), 'w').close()
1820 self.git('annex add c', cwd=self.srcdir) 1996 self.git('annex add c', cwd=self.srcdir)
1821 self.git('commit -m annex-c -a', cwd=self.srcdir) 1997 self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir)
1822 bb.process.run('chmod u+w -R %s' % os.path.join(self.srcdir, '.git', 'annex')) 1998 bb.process.run('chmod u+w -R %s' % self.srcdir)
1823 1999
1824 uri = 'gitannex://%s;protocol=file;subdir=${S}' % self.srcdir 2000 uri = 'gitannex://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
1825 fetcher, ud = self.fetch_shallow(uri) 2001 fetcher, ud = self.fetch_shallow(uri)
1826 2002
1827 self.assertRevCount(1) 2003 self.assertRevCount(1)
@@ -1910,9 +2086,9 @@ class GitShallowTest(FetcherTest):
1910 # Set up the mirror 2086 # Set up the mirror
1911 mirrordir = os.path.join(self.tempdir, 'mirror') 2087 mirrordir = os.path.join(self.tempdir, 'mirror')
1912 bb.utils.mkdirhier(mirrordir) 2088 bb.utils.mkdirhier(mirrordir)
1913 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir) 2089 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % mirrordir)
1914 2090
1915 os.rename(os.path.join(self.dldir, mirrortarball), 2091 bb.utils.rename(os.path.join(self.dldir, mirrortarball),
1916 os.path.join(mirrordir, mirrortarball)) 2092 os.path.join(mirrordir, mirrortarball))
1917 2093
1918 # Fetch from the mirror 2094 # Fetch from the mirror
@@ -2033,7 +2209,7 @@ class GitShallowTest(FetcherTest):
2033 2209
2034 @skipIfNoNetwork() 2210 @skipIfNoNetwork()
2035 def test_bitbake(self): 2211 def test_bitbake(self):
2036 self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir) 2212 self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir)
2037 self.git('config core.bare true', cwd=self.srcdir) 2213 self.git('config core.bare true', cwd=self.srcdir)
2038 self.git('fetch', cwd=self.srcdir) 2214 self.git('fetch', cwd=self.srcdir)
2039 2215
@@ -2068,7 +2244,7 @@ class GitShallowTest(FetcherTest):
2068 self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0') 2244 self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0')
2069 self.d.setVar('BB_GIT_SHALLOW', '1') 2245 self.d.setVar('BB_GIT_SHALLOW', '1')
2070 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 2246 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
2071 fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests"], self.d) 2247 fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests;branch=master;protocol=https"], self.d)
2072 fetcher.download() 2248 fetcher.download()
2073 2249
2074 bb.utils.remove(self.dldir + "/*.tar.gz") 2250 bb.utils.remove(self.dldir + "/*.tar.gz")
@@ -2078,12 +2254,18 @@ class GitShallowTest(FetcherTest):
2078 self.assertIn("fstests.doap", dir) 2254 self.assertIn("fstests.doap", dir)
2079 2255
2080class GitLfsTest(FetcherTest): 2256class GitLfsTest(FetcherTest):
2257 def skipIfNoGitLFS():
2258 import shutil
2259 if not shutil.which('git-lfs'):
2260 return unittest.skip('git-lfs not installed')
2261 return lambda f: f
2262
2081 def setUp(self): 2263 def setUp(self):
2082 FetcherTest.setUp(self) 2264 FetcherTest.setUp(self)
2083 2265
2084 self.gitdir = os.path.join(self.tempdir, 'git') 2266 self.gitdir = os.path.join(self.tempdir, 'git')
2085 self.srcdir = os.path.join(self.tempdir, 'gitsource') 2267 self.srcdir = os.path.join(self.tempdir, 'gitsource')
2086 2268
2087 self.d.setVar('WORKDIR', self.tempdir) 2269 self.d.setVar('WORKDIR', self.tempdir)
2088 self.d.setVar('S', self.gitdir) 2270 self.d.setVar('S', self.gitdir)
2089 self.d.delVar('PREMIRRORS') 2271 self.d.delVar('PREMIRRORS')
@@ -2091,22 +2273,18 @@ class GitLfsTest(FetcherTest):
2091 2273
2092 self.d.setVar('SRCREV', '${AUTOREV}') 2274 self.d.setVar('SRCREV', '${AUTOREV}')
2093 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 2275 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
2276 self.d.setVar("__BBSRCREV_SEEN", "1")
2094 2277
2095 bb.utils.mkdirhier(self.srcdir) 2278 bb.utils.mkdirhier(self.srcdir)
2096 self.git('init', cwd=self.srcdir) 2279 self.git_init(cwd=self.srcdir)
2097 with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs: 2280 self.commit_file('.gitattributes', '*.mp3 filter=lfs -text')
2098 attrs.write('*.mp3 filter=lfs -text')
2099 self.git(['add', '.gitattributes'], cwd=self.srcdir)
2100 self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir)
2101 2281
2102 def git(self, cmd, cwd=None): 2282 def commit_file(self, filename, content):
2103 if isinstance(cmd, str): 2283 with open(os.path.join(self.srcdir, filename), "w") as f:
2104 cmd = 'git ' + cmd 2284 f.write(content)
2105 else: 2285 self.git(["add", filename], cwd=self.srcdir)
2106 cmd = ['git'] + cmd 2286 self.git(["commit", "-m", "Change"], cwd=self.srcdir)
2107 if cwd is None: 2287 return self.git(["rev-parse", "HEAD"], cwd=self.srcdir).strip()
2108 cwd = self.gitdir
2109 return bb.process.run(cmd, cwd=cwd)[0]
2110 2288
2111 def fetch(self, uri=None, download=True): 2289 def fetch(self, uri=None, download=True):
2112 uris = self.d.getVar('SRC_URI').split() 2290 uris = self.d.getVar('SRC_URI').split()
@@ -2119,65 +2297,158 @@ class GitLfsTest(FetcherTest):
2119 ud = fetcher.ud[uri] 2297 ud = fetcher.ud[uri]
2120 return fetcher, ud 2298 return fetcher, ud
2121 2299
2300 def get_real_git_lfs_file(self):
2301 self.d.setVar('PATH', os.environ.get('PATH'))
2302 fetcher, ud = self.fetch()
2303 fetcher.unpack(self.d.getVar('WORKDIR'))
2304 unpacked_lfs_file = os.path.join(self.d.getVar('WORKDIR'), 'git', "Cat_poster_1.jpg")
2305 return unpacked_lfs_file
2306
2307 @skipIfNoGitLFS()
2308 def test_fetch_lfs_on_srcrev_change(self):
2309 """Test if fetch downloads missing LFS objects when a different revision within an existing repository is requested"""
2310 self.git(["lfs", "install", "--local"], cwd=self.srcdir)
2311
2312 @contextlib.contextmanager
2313 def hide_upstream_repository():
2314 """Hide the upstream repository to make sure that git lfs cannot pull from it"""
2315 temp_name = self.srcdir + ".bak"
2316 os.rename(self.srcdir, temp_name)
2317 try:
2318 yield
2319 finally:
2320 os.rename(temp_name, self.srcdir)
2321
2322 def fetch_and_verify(revision, filename, content):
2323 self.d.setVar('SRCREV', revision)
2324 fetcher, ud = self.fetch()
2325
2326 with hide_upstream_repository():
2327 workdir = self.d.getVar('WORKDIR')
2328 fetcher.unpack(workdir)
2329
2330 with open(os.path.join(workdir, "git", filename)) as f:
2331 self.assertEqual(f.read(), content)
2332
2333 commit_1 = self.commit_file("a.mp3", "version 1")
2334 commit_2 = self.commit_file("a.mp3", "version 2")
2335
2336 self.d.setVar('SRC_URI', "git://%s;protocol=file;lfs=1;branch=master" % self.srcdir)
2337
2338 # Seed the local download folder by fetching the latest commit and verifying that the LFS contents are
2339 # available even when the upstream repository disappears.
2340 fetch_and_verify(commit_2, "a.mp3", "version 2")
2341 # Verify that even when an older revision is fetched, the needed LFS objects are fetched into the download
2342 # folder.
2343 fetch_and_verify(commit_1, "a.mp3", "version 1")
2344
2345 @skipIfNoGitLFS()
2346 @skipIfNoNetwork()
2347 def test_real_git_lfs_repo_succeeds_without_lfs_param(self):
2348 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master")
2349 f = self.get_real_git_lfs_file()
2350 self.assertTrue(os.path.exists(f))
2351 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f))
2352
2353 @skipIfNoGitLFS()
2354 @skipIfNoNetwork()
2355 def test_real_git_lfs_repo_succeeds(self):
2356 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=1")
2357 f = self.get_real_git_lfs_file()
2358 self.assertTrue(os.path.exists(f))
2359 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f))
2360
2361 @skipIfNoGitLFS()
2362 @skipIfNoNetwork()
2363 def test_real_git_lfs_repo_skips(self):
2364 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=0")
2365 f = self.get_real_git_lfs_file()
2366 # This is the actual non-smudged placeholder file on the repo if git-lfs does not run
2367 lfs_file = (
2368 'version https://git-lfs.github.com/spec/v1\n'
2369 'oid sha256:34be66b1a39a1955b46a12588df9d5f6fc1da790e05cf01f3c7422f4bbbdc26b\n'
2370 'size 11423554\n'
2371 )
2372
2373 with open(f) as fh:
2374 self.assertEqual(lfs_file, fh.read())
2375
2376 @skipIfNoGitLFS()
2122 def test_lfs_enabled(self): 2377 def test_lfs_enabled(self):
2123 import shutil 2378 import shutil
2124 2379
2125 uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir 2380 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir
2126 self.d.setVar('SRC_URI', uri) 2381 self.d.setVar('SRC_URI', uri)
2127 2382
2128 # Careful: suppress initial attempt at downloading until 2383 # With git-lfs installed, test that we can fetch and unpack
2129 # we know whether git-lfs is installed. 2384 fetcher, ud = self.fetch()
2130 fetcher, ud = self.fetch(uri=None, download=False) 2385 shutil.rmtree(self.gitdir, ignore_errors=True)
2131 self.assertIsNotNone(ud.method._find_git_lfs) 2386 fetcher.unpack(self.d.getVar('WORKDIR'))
2132
2133 # If git-lfs can be found, the unpack should be successful. Only
2134 # attempt this with the real live copy of git-lfs installed.
2135 if ud.method._find_git_lfs(self.d):
2136 fetcher.download()
2137 shutil.rmtree(self.gitdir, ignore_errors=True)
2138 fetcher.unpack(self.d.getVar('WORKDIR'))
2139
2140 # If git-lfs cannot be found, the unpack should throw an error
2141 with self.assertRaises(bb.fetch2.FetchError):
2142 fetcher.download()
2143 ud.method._find_git_lfs = lambda d: False
2144 shutil.rmtree(self.gitdir, ignore_errors=True)
2145 fetcher.unpack(self.d.getVar('WORKDIR'))
2146 2387
2388 @skipIfNoGitLFS()
2147 def test_lfs_disabled(self): 2389 def test_lfs_disabled(self):
2148 import shutil 2390 import shutil
2149 2391
2150 uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir 2392 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir
2151 self.d.setVar('SRC_URI', uri) 2393 self.d.setVar('SRC_URI', uri)
2152 2394
2153 # In contrast to test_lfs_enabled(), allow the implicit download 2395 # Verify that the fetcher can survive even if the source
2154 # done by self.fetch() to occur here. The point of this test case
2155 # is to verify that the fetcher can survive even if the source
2156 # repository has Git LFS usage configured. 2396 # repository has Git LFS usage configured.
2157 fetcher, ud = self.fetch() 2397 fetcher, ud = self.fetch()
2158 self.assertIsNotNone(ud.method._find_git_lfs)
2159
2160 # If git-lfs can be found, the unpack should be successful. A
2161 # live copy of git-lfs is not required for this case, so
2162 # unconditionally forge its presence.
2163 ud.method._find_git_lfs = lambda d: True
2164 shutil.rmtree(self.gitdir, ignore_errors=True)
2165 fetcher.unpack(self.d.getVar('WORKDIR')) 2398 fetcher.unpack(self.d.getVar('WORKDIR'))
2166 2399
2167 # If git-lfs cannot be found, the unpack should be successful 2400 def test_lfs_enabled_not_installed(self):
2168 ud.method._find_git_lfs = lambda d: False 2401 import shutil
2169 shutil.rmtree(self.gitdir, ignore_errors=True) 2402
2170 fetcher.unpack(self.d.getVar('WORKDIR')) 2403 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir
2404 self.d.setVar('SRC_URI', uri)
2405
2406 # Careful: suppress initial attempt at downloading
2407 fetcher, ud = self.fetch(uri=None, download=False)
2408
2409 # Artificially assert that git-lfs is not installed, so
2410 # we can verify a failure to unpack in it's absence.
2411 old_find_git_lfs = ud.method._find_git_lfs
2412 try:
2413 # If git-lfs cannot be found, the unpack should throw an error
2414 with self.assertRaises(bb.fetch2.FetchError):
2415 fetcher.download()
2416 ud.method._find_git_lfs = lambda d: False
2417 shutil.rmtree(self.gitdir, ignore_errors=True)
2418 fetcher.unpack(self.d.getVar('WORKDIR'))
2419 finally:
2420 ud.method._find_git_lfs = old_find_git_lfs
2421
2422 def test_lfs_disabled_not_installed(self):
2423 import shutil
2424
2425 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir
2426 self.d.setVar('SRC_URI', uri)
2427
2428 # Careful: suppress initial attempt at downloading
2429 fetcher, ud = self.fetch(uri=None, download=False)
2430
2431 # Artificially assert that git-lfs is not installed, so
2432 # we can verify a failure to unpack in it's absence.
2433 old_find_git_lfs = ud.method._find_git_lfs
2434 try:
2435 # Even if git-lfs cannot be found, the unpack should be successful
2436 fetcher.download()
2437 ud.method._find_git_lfs = lambda d: False
2438 shutil.rmtree(self.gitdir, ignore_errors=True)
2439 fetcher.unpack(self.d.getVar('WORKDIR'))
2440 finally:
2441 ud.method._find_git_lfs = old_find_git_lfs
2171 2442
2172class GitURLWithSpacesTest(FetcherTest): 2443class GitURLWithSpacesTest(FetcherTest):
2173 test_git_urls = { 2444 test_git_urls = {
2174 "git://tfs-example.org:22/tfs/example%20path/example.git" : { 2445 "git://tfs-example.org:22/tfs/example%20path/example.git;branch=master" : {
2175 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git', 2446 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git;branch=master',
2176 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git', 2447 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git',
2177 'path': '/tfs/example path/example.git' 2448 'path': '/tfs/example path/example.git'
2178 }, 2449 },
2179 "git://tfs-example.org:22/tfs/example%20path/example%20repo.git" : { 2450 "git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master" : {
2180 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git', 2451 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master',
2181 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git', 2452 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git',
2182 'path': '/tfs/example path/example repo.git' 2453 'path': '/tfs/example path/example repo.git'
2183 } 2454 }
@@ -2201,11 +2472,129 @@ class GitURLWithSpacesTest(FetcherTest):
2201 self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2472 self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname']))
2202 self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz')) 2473 self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz'))
2203 2474
2475class CrateTest(FetcherTest):
2476 @skipIfNoNetwork()
2477 def test_crate_url(self):
2478
2479 uri = "crate://crates.io/glob/0.2.11"
2480 self.d.setVar('SRC_URI', uri)
2481
2482 uris = self.d.getVar('SRC_URI').split()
2483 d = self.d
2484
2485 fetcher = bb.fetch2.Fetch(uris, self.d)
2486 ud = fetcher.ud[fetcher.urls[0]]
2487
2488 self.assertIn("name", ud.parm)
2489 self.assertEqual(ud.parm["name"], "glob-0.2.11")
2490 self.assertIn("downloadfilename", ud.parm)
2491 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
2492
2493 fetcher.download()
2494 fetcher.unpack(self.tempdir)
2495 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked'])
2496 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done'])
2497 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json"))
2498 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs"))
2499
2500 @skipIfNoNetwork()
2501 def test_crate_url_matching_recipe(self):
2502
2503 self.d.setVar('BP', 'glob-0.2.11')
2504
2505 uri = "crate://crates.io/glob/0.2.11"
2506 self.d.setVar('SRC_URI', uri)
2507
2508 uris = self.d.getVar('SRC_URI').split()
2509 d = self.d
2510
2511 fetcher = bb.fetch2.Fetch(uris, self.d)
2512 ud = fetcher.ud[fetcher.urls[0]]
2513
2514 self.assertIn("name", ud.parm)
2515 self.assertEqual(ud.parm["name"], "glob-0.2.11")
2516 self.assertIn("downloadfilename", ud.parm)
2517 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
2518
2519 fetcher.download()
2520 fetcher.unpack(self.tempdir)
2521 self.assertEqual(sorted(os.listdir(self.tempdir)), ['download', 'glob-0.2.11', 'unpacked'])
2522 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done'])
2523 self.assertTrue(os.path.exists(self.tempdir + "/glob-0.2.11/src/lib.rs"))
2524
2525 @skipIfNoNetwork()
2526 def test_crate_url_params(self):
2527
2528 uri = "crate://crates.io/aho-corasick/0.7.20;name=aho-corasick-renamed"
2529 self.d.setVar('SRC_URI', uri)
2530
2531 uris = self.d.getVar('SRC_URI').split()
2532 d = self.d
2533
2534 fetcher = bb.fetch2.Fetch(uris, self.d)
2535 ud = fetcher.ud[fetcher.urls[0]]
2536
2537 self.assertIn("name", ud.parm)
2538 self.assertEqual(ud.parm["name"], "aho-corasick-renamed")
2539 self.assertIn("downloadfilename", ud.parm)
2540 self.assertEqual(ud.parm["downloadfilename"], "aho-corasick-0.7.20.crate")
2541
2542 fetcher.download()
2543 fetcher.unpack(self.tempdir)
2544 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked'])
2545 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['aho-corasick-0.7.20.crate', 'aho-corasick-0.7.20.crate.done'])
2546 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/.cargo-checksum.json"))
2547 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/src/lib.rs"))
2548
2549 @skipIfNoNetwork()
2550 def test_crate_url_multi(self):
2551
2552 uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35"
2553 self.d.setVar('SRC_URI', uri)
2554
2555 uris = self.d.getVar('SRC_URI').split()
2556 d = self.d
2557
2558 fetcher = bb.fetch2.Fetch(uris, self.d)
2559 ud = fetcher.ud[fetcher.urls[0]]
2560
2561 self.assertIn("name", ud.parm)
2562 self.assertEqual(ud.parm["name"], "glob-0.2.11")
2563 self.assertIn("downloadfilename", ud.parm)
2564 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
2565
2566 ud = fetcher.ud[fetcher.urls[1]]
2567 self.assertIn("name", ud.parm)
2568 self.assertEqual(ud.parm["name"], "time-0.1.35")
2569 self.assertIn("downloadfilename", ud.parm)
2570 self.assertEqual(ud.parm["downloadfilename"], "time-0.1.35.crate")
2571
2572 fetcher.download()
2573 fetcher.unpack(self.tempdir)
2574 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked'])
2575 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done', 'time-0.1.35.crate', 'time-0.1.35.crate.done'])
2576 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json"))
2577 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs"))
2578 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json"))
2579 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs"))
2580
2581 @skipIfNoNetwork()
2582 def test_crate_incorrect_cksum(self):
2583 uri = "crate://crates.io/aho-corasick/0.7.20"
2584 self.d.setVar('SRC_URI', uri)
2585 self.d.setVarFlag("SRC_URI", "aho-corasick-0.7.20.sha256sum", hashlib.sha256("Invalid".encode("utf-8")).hexdigest())
2586
2587 uris = self.d.getVar('SRC_URI').split()
2588
2589 fetcher = bb.fetch2.Fetch(uris, self.d)
2590 with self.assertRaisesRegex(bb.fetch2.FetchError, "Fetcher failure for URL"):
2591 fetcher.download()
2592
2204class NPMTest(FetcherTest): 2593class NPMTest(FetcherTest):
2205 def skipIfNoNpm(): 2594 def skipIfNoNpm():
2206 import shutil 2595 import shutil
2207 if not shutil.which('npm'): 2596 if not shutil.which('npm'):
2208 return unittest.skip('npm not installed, tests being skipped') 2597 return unittest.skip('npm not installed')
2209 return lambda f: f 2598 return lambda f: f
2210 2599
2211 @skipIfNoNpm() 2600 @skipIfNoNpm()
@@ -2250,11 +2639,42 @@ class NPMTest(FetcherTest):
2250 ud = fetcher.ud[fetcher.urls[0]] 2639 ud = fetcher.ud[fetcher.urls[0]]
2251 fetcher.download() 2640 fetcher.download()
2252 self.assertTrue(os.path.exists(ud.localpath)) 2641 self.assertTrue(os.path.exists(ud.localpath))
2642
2643 # Setup the mirror by renaming the download directory
2644 mirrordir = os.path.join(self.tempdir, 'mirror')
2645 bb.utils.rename(self.dldir, mirrordir)
2646 os.mkdir(self.dldir)
2647
2648 # Configure the premirror to be used
2649 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/npm2' % mirrordir)
2650 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
2651
2652 # Fetch again
2653 self.assertFalse(os.path.exists(ud.localpath))
2654 # The npm fetcher doesn't handle that the .resolved file disappears
2655 # while the fetcher object exists, which it does when we rename the
2656 # download directory to "mirror" above. Thus we need a new fetcher to go
2657 # with the now empty download directory.
2658 fetcher = bb.fetch.Fetch([url], self.d)
2659 ud = fetcher.ud[fetcher.urls[0]]
2660 fetcher.download()
2661 self.assertTrue(os.path.exists(ud.localpath))
2662
2663 @skipIfNoNpm()
2664 @skipIfNoNetwork()
2665 def test_npm_premirrors_with_specified_filename(self):
2666 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2667 # Fetch once to get a tarball
2668 fetcher = bb.fetch.Fetch([url], self.d)
2669 ud = fetcher.ud[fetcher.urls[0]]
2670 fetcher.download()
2671 self.assertTrue(os.path.exists(ud.localpath))
2253 # Setup the mirror 2672 # Setup the mirror
2254 mirrordir = os.path.join(self.tempdir, 'mirror') 2673 mirrordir = os.path.join(self.tempdir, 'mirror')
2255 bb.utils.mkdirhier(mirrordir) 2674 bb.utils.mkdirhier(mirrordir)
2256 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 2675 mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath))
2257 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir) 2676 os.replace(ud.localpath, mirrorfilename)
2677 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s' % mirrorfilename)
2258 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2678 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
2259 # Fetch again 2679 # Fetch again
2260 self.assertFalse(os.path.exists(ud.localpath)) 2680 self.assertFalse(os.path.exists(ud.localpath))
@@ -2274,7 +2694,7 @@ class NPMTest(FetcherTest):
2274 mirrordir = os.path.join(self.tempdir, 'mirror') 2694 mirrordir = os.path.join(self.tempdir, 'mirror')
2275 bb.utils.mkdirhier(mirrordir) 2695 bb.utils.mkdirhier(mirrordir)
2276 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 2696 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
2277 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir) 2697 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir)
2278 # Update the resolved url to an invalid url 2698 # Update the resolved url to an invalid url
2279 with open(ud.resolvefile, 'r') as f: 2699 with open(ud.resolvefile, 'r') as f:
2280 url = f.read() 2700 url = f.read()
@@ -2293,7 +2713,7 @@ class NPMTest(FetcherTest):
2293 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz' 2713 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz'
2294 fetcher = bb.fetch.Fetch([url], self.d) 2714 fetcher = bb.fetch.Fetch([url], self.d)
2295 fetcher.download() 2715 fetcher.download()
2296 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'foo-bar.tgz'))) 2716 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'foo-bar.tgz')))
2297 fetcher.unpack(self.unpackdir) 2717 fetcher.unpack(self.unpackdir)
2298 unpackdir = os.path.join(self.unpackdir, 'foo', 'bar') 2718 unpackdir = os.path.join(self.unpackdir, 'foo', 'bar')
2299 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2719 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
@@ -2323,7 +2743,7 @@ class NPMTest(FetcherTest):
2323 @skipIfNoNpm() 2743 @skipIfNoNpm()
2324 @skipIfNoNetwork() 2744 @skipIfNoNetwork()
2325 def test_npm_registry_alternate(self): 2745 def test_npm_registry_alternate(self):
2326 url = 'npm://registry.freajs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2746 url = 'npm://skimdb.npmjs.com;package=@savoirfairelinux/node-server-example;version=1.0.0'
2327 fetcher = bb.fetch.Fetch([url], self.d) 2747 fetcher = bb.fetch.Fetch([url], self.d)
2328 fetcher.download() 2748 fetcher.download()
2329 fetcher.unpack(self.unpackdir) 2749 fetcher.unpack(self.unpackdir)
@@ -2433,6 +2853,45 @@ class NPMTest(FetcherTest):
2433 2853
2434 @skipIfNoNpm() 2854 @skipIfNoNpm()
2435 @skipIfNoNetwork() 2855 @skipIfNoNetwork()
2856 def test_npmsw_git(self):
2857 swfile = self.create_shrinkwrap_file({
2858 'dependencies': {
2859 'cookie': {
2860 'version': 'github:jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09',
2861 'from': 'github:jshttp/cookie.git'
2862 }
2863 }
2864 })
2865 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2866 fetcher.download()
2867 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git')))
2868
2869 swfile = self.create_shrinkwrap_file({
2870 'dependencies': {
2871 'cookie': {
2872 'version': 'jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09',
2873 'from': 'jshttp/cookie.git'
2874 }
2875 }
2876 })
2877 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2878 fetcher.download()
2879 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git')))
2880
2881 swfile = self.create_shrinkwrap_file({
2882 'dependencies': {
2883 'nodejs': {
2884 'version': 'gitlab:gitlab-examples/nodejs.git#892a1f16725e56cc3a2cb0d677be42935c8fc262',
2885 'from': 'gitlab:gitlab-examples/nodejs'
2886 }
2887 }
2888 })
2889 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2890 fetcher.download()
2891 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'gitlab.com.gitlab-examples.nodejs.git')))
2892
2893 @skipIfNoNpm()
2894 @skipIfNoNetwork()
2436 def test_npmsw_dev(self): 2895 def test_npmsw_dev(self):
2437 swfile = self.create_shrinkwrap_file({ 2896 swfile = self.create_shrinkwrap_file({
2438 'dependencies': { 2897 'dependencies': {
@@ -2590,7 +3049,7 @@ class NPMTest(FetcherTest):
2590 mirrordir = os.path.join(self.tempdir, 'mirror') 3049 mirrordir = os.path.join(self.tempdir, 'mirror')
2591 bb.utils.mkdirhier(mirrordir) 3050 bb.utils.mkdirhier(mirrordir)
2592 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 3051 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
2593 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir) 3052 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir)
2594 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 3053 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
2595 # Fetch again 3054 # Fetch again
2596 self.assertFalse(os.path.exists(ud.localpath)) 3055 self.assertFalse(os.path.exists(ud.localpath))
@@ -2619,7 +3078,7 @@ class NPMTest(FetcherTest):
2619 mirrordir = os.path.join(self.tempdir, 'mirror') 3078 mirrordir = os.path.join(self.tempdir, 'mirror')
2620 bb.utils.mkdirhier(mirrordir) 3079 bb.utils.mkdirhier(mirrordir)
2621 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 3080 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
2622 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir) 3081 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir)
2623 # Fetch again with invalid url 3082 # Fetch again with invalid url
2624 self.assertFalse(os.path.exists(ud.localpath)) 3083 self.assertFalse(os.path.exists(ud.localpath))
2625 swfile = self.create_shrinkwrap_file({ 3084 swfile = self.create_shrinkwrap_file({
@@ -2634,3 +3093,273 @@ class NPMTest(FetcherTest):
2634 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3093 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2635 fetcher.download() 3094 fetcher.download()
2636 self.assertTrue(os.path.exists(ud.localpath)) 3095 self.assertTrue(os.path.exists(ud.localpath))
3096
3097class GitSharedTest(FetcherTest):
3098 def setUp(self):
3099 super(GitSharedTest, self).setUp()
3100 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
3101 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
3102 self.d.setVar("__BBSRCREV_SEEN", "1")
3103
3104 @skipIfNoNetwork()
3105 def test_shared_unpack(self):
3106 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3107
3108 fetcher.download()
3109 fetcher.unpack(self.unpackdir)
3110 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
3111 self.assertTrue(os.path.exists(alt))
3112
3113 @skipIfNoNetwork()
3114 def test_noshared_unpack(self):
3115 self.d.setVar('BB_GIT_NOSHARED', '1')
3116 self.unpackdir += '_noshared'
3117 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3118
3119 fetcher.download()
3120 fetcher.unpack(self.unpackdir)
3121 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
3122 self.assertFalse(os.path.exists(alt))
3123
3124
3125class FetchPremirroronlyLocalTest(FetcherTest):
3126
3127 def setUp(self):
3128 super(FetchPremirroronlyLocalTest, self).setUp()
3129 self.mirrordir = os.path.join(self.tempdir, "mirrors")
3130 os.mkdir(self.mirrordir)
3131 self.reponame = "bitbake"
3132 self.gitdir = os.path.join(self.tempdir, "git", self.reponame)
3133 self.recipe_url = "git://git.fake.repo/bitbake;branch=master;protocol=https"
3134 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
3135 self.d.setVar("BB_NO_NETWORK", "1")
3136 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
3137 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz"
3138 self.mirrorfile = os.path.join(self.mirrordir, self.mirrorname)
3139 self.testfilename = "bitbake-fetch.test"
3140
3141 def make_git_repo(self):
3142 recipeurl = "git:/git.fake.repo/bitbake"
3143 os.makedirs(self.gitdir)
3144 self.git_init(cwd=self.gitdir)
3145 for i in range(0):
3146 self.git_new_commit()
3147 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir)
3148
3149 def git_new_commit(self):
3150 import random
3151 os.unlink(os.path.join(self.mirrordir, self.mirrorname))
3152 branch = self.git("branch --show-current", self.gitdir).split()
3153 with open(os.path.join(self.gitdir, self.testfilename), "w") as testfile:
3154 testfile.write("File {} from branch {}; Useless random data {}".format(self.testfilename, branch, random.random()))
3155 self.git("add {}".format(self.testfilename), self.gitdir)
3156 self.git("commit -a -m \"This random commit {} in branch {}. I'm useless.\"".format(random.random(), branch), self.gitdir)
3157 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir)
3158 return self.git("rev-parse HEAD", self.gitdir).strip()
3159
3160 def git_new_branch(self, name):
3161 self.git_new_commit()
3162 head = self.git("rev-parse HEAD", self.gitdir).strip()
3163 self.git("checkout -b {}".format(name), self.gitdir)
3164 newrev = self.git_new_commit()
3165 self.git("checkout {}".format(head), self.gitdir)
3166 return newrev
3167
3168 def test_mirror_multiple_fetches(self):
3169 self.make_git_repo()
3170 self.d.setVar("SRCREV", self.git_new_commit())
3171 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3172 fetcher.download()
3173 fetcher.unpack(self.unpackdir)
3174 ## New commit in premirror. it's not in the download_dir
3175 self.d.setVar("SRCREV", self.git_new_commit())
3176 fetcher2 = bb.fetch.Fetch([self.recipe_url], self.d)
3177 fetcher2.download()
3178 fetcher2.unpack(self.unpackdir)
3179 ## New commit in premirror. it's not in the download_dir
3180 self.d.setVar("SRCREV", self.git_new_commit())
3181 fetcher3 = bb.fetch.Fetch([self.recipe_url], self.d)
3182 fetcher3.download()
3183 fetcher3.unpack(self.unpackdir)
3184
3185
3186 def test_mirror_commit_nonexistent(self):
3187 self.make_git_repo()
3188 self.d.setVar("SRCREV", "0"*40)
3189 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3190 with self.assertRaises(bb.fetch2.NetworkAccess):
3191 fetcher.download()
3192
3193 def test_mirror_commit_exists(self):
3194 self.make_git_repo()
3195 self.d.setVar("SRCREV", self.git_new_commit())
3196 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3197 fetcher.download()
3198 fetcher.unpack(self.unpackdir)
3199
3200 def test_mirror_tarball_nonexistent(self):
3201 self.d.setVar("SRCREV", "0"*40)
3202 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3203 with self.assertRaises(bb.fetch2.NetworkAccess):
3204 fetcher.download()
3205
3206 def test_mirror_tarball_multiple_branches(self):
3207 """
3208 test if PREMIRRORS can handle multiple name/branches correctly
3209 both branches have required revisions
3210 """
3211 self.make_git_repo()
3212 branch1rev = self.git_new_branch("testbranch1")
3213 branch2rev = self.git_new_branch("testbranch2")
3214 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2"
3215 self.d.setVar("SRCREV_branch1", branch1rev)
3216 self.d.setVar("SRCREV_branch2", branch2rev)
3217 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3218 self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist")
3219 fetcher.download()
3220 fetcher.unpack(os.path.join(self.tempdir, "unpacked"))
3221 unpacked = os.path.join(self.tempdir, "unpacked", "git", self.testfilename)
3222 self.assertTrue(os.path.exists(unpacked), "Repo has not been unpackaged properly!")
3223 with open(unpacked, 'r') as f:
3224 content = f.read()
3225 ## We expect to see testbranch1 in the file, not master, not testbranch2
3226 self.assertTrue(content.find("testbranch1") != -1, "Wrong branch has been checked out!")
3227
3228 def test_mirror_tarball_multiple_branches_nobranch(self):
3229 """
3230 test if PREMIRRORS can handle multiple name/branches correctly
3231 Unbalanced name/branches raises ParameterError
3232 """
3233 self.make_git_repo()
3234 branch1rev = self.git_new_branch("testbranch1")
3235 branch2rev = self.git_new_branch("testbranch2")
3236 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1;protocol=https;name=branch1,branch2"
3237 self.d.setVar("SRCREV_branch1", branch1rev)
3238 self.d.setVar("SRCREV_branch2", branch2rev)
3239 with self.assertRaises(bb.fetch2.ParameterError):
3240 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3241
3242 def test_mirror_tarball_multiple_branches_norev(self):
3243 """
3244 test if PREMIRRORS can handle multiple name/branches correctly
3245 one of the branches specifies non existing SRCREV
3246 """
3247 self.make_git_repo()
3248 branch1rev = self.git_new_branch("testbranch1")
3249 branch2rev = self.git_new_branch("testbranch2")
3250 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2"
3251 self.d.setVar("SRCREV_branch1", branch1rev)
3252 self.d.setVar("SRCREV_branch2", "0"*40)
3253 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3254 self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist")
3255 with self.assertRaises(bb.fetch2.NetworkAccess):
3256 fetcher.download()
3257
3258
3259class FetchPremirroronlyNetworkTest(FetcherTest):
3260
3261 def setUp(self):
3262 super(FetchPremirroronlyNetworkTest, self).setUp()
3263 self.mirrordir = os.path.join(self.tempdir, "mirrors")
3264 os.mkdir(self.mirrordir)
3265 self.reponame = "fstests"
3266 self.clonedir = os.path.join(self.tempdir, "git")
3267 self.gitdir = os.path.join(self.tempdir, "git", "{}.git".format(self.reponame))
3268 self.recipe_url = "git://git.yoctoproject.org/fstests;protocol=https"
3269 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
3270 self.d.setVar("BB_NO_NETWORK", "0")
3271 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
3272
3273 def make_git_repo(self):
3274 import shutil
3275 self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz"
3276 os.makedirs(self.clonedir)
3277 self.git("clone --bare --shallow-since=\"01.01.2013\" {}".format(self.recipe_url), self.clonedir)
3278 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir)
3279 shutil.rmtree(self.clonedir)
3280
3281 @skipIfNoNetwork()
3282 def test_mirror_tarball_updated(self):
3283 self.make_git_repo()
3284 ## Upstream commit is in the mirror
3285 self.d.setVar("SRCREV", "49d65d53c2bf558ae6e9185af0f3af7b79d255ec")
3286 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3287 fetcher.download()
3288
3289 @skipIfNoNetwork()
3290 def test_mirror_tarball_outdated(self):
3291 self.make_git_repo()
3292 ## Upstream commit not in the mirror
3293 self.d.setVar("SRCREV", "15413486df1f5a5b5af699b6f3ba5f0984e52a9f")
3294 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3295 with self.assertRaises(bb.fetch2.NetworkAccess):
3296 fetcher.download()
3297
3298class FetchPremirroronlyMercurialTest(FetcherTest):
3299 """ Test for premirrors with mercurial repos
3300 the test covers also basic hg:// clone (see fetch_and_create_tarball
3301 """
3302 def skipIfNoHg():
3303 import shutil
3304 if not shutil.which('hg'):
3305 return unittest.skip('Mercurial not installed')
3306 return lambda f: f
3307
3308 def setUp(self):
3309 super(FetchPremirroronlyMercurialTest, self).setUp()
3310 self.mirrordir = os.path.join(self.tempdir, "mirrors")
3311 os.mkdir(self.mirrordir)
3312 self.reponame = "libgnt"
3313 self.clonedir = os.path.join(self.tempdir, "hg")
3314 self.recipe_url = "hg://keep.imfreedom.org/libgnt;module=libgnt"
3315 self.d.setVar("SRCREV", "53e8b422faaf")
3316 self.mirrorname = "hg_libgnt_keep.imfreedom.org_.libgnt.tar.gz"
3317
3318 def fetch_and_create_tarball(self):
3319 """
3320 Ask bitbake to download repo and prepare mirror tarball for us
3321 """
3322 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
3323 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3324 fetcher.download()
3325 mirrorfile = os.path.join(self.d.getVar("DL_DIR"), self.mirrorname)
3326 self.assertTrue(os.path.exists(mirrorfile), "Mirror tarball {} has not been created".format(mirrorfile))
3327 ## moving tarball to mirror directory
3328 os.rename(mirrorfile, os.path.join(self.mirrordir, self.mirrorname))
3329 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "0")
3330
3331
3332 @skipIfNoNetwork()
3333 @skipIfNoHg()
3334 def test_premirror_mercurial(self):
3335 self.fetch_and_create_tarball()
3336 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
3337 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
3338 self.d.setVar("BB_NO_NETWORK", "1")
3339 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3340 fetcher.download()
3341
3342class FetchPremirroronlyBrokenTarball(FetcherTest):
3343
3344 def setUp(self):
3345 super(FetchPremirroronlyBrokenTarball, self).setUp()
3346 self.mirrordir = os.path.join(self.tempdir, "mirrors")
3347 os.mkdir(self.mirrordir)
3348 self.reponame = "bitbake"
3349 self.gitdir = os.path.join(self.tempdir, "git", self.reponame)
3350 self.recipe_url = "git://git.fake.repo/bitbake;protocol=https"
3351 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
3352 self.d.setVar("BB_NO_NETWORK", "1")
3353 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
3354 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz"
3355 with open(os.path.join(self.mirrordir, self.mirrorname), 'w') as targz:
3356 targz.write("This is not tar.gz file!")
3357
3358 def test_mirror_broken_download(self):
3359 import sys
3360 self.d.setVar("SRCREV", "0"*40)
3361 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3362 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs() as logs:
3363 fetcher.download()
3364 output = "".join(logs.output)
3365 self.assertFalse(" not a git repository (or any parent up to mount point /)" in output)
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2024-05-05 00:59:13 +0000