summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorMark Hatle <mark.hatle@windriver.com>2019-05-17 04:54:50 -0400
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-05-21 12:56:34 +0100
commita1c1f3f7c583267ef9ccc9b24c84568ed0cb47f6 (patch)
treec4500d76aaeeed8c142f8905324e301b9acad29f /bitbake/lib/bb
parent937e67f95e7ea54a18927826c4888515b67436d6 (diff)
downloadpoky-a1c1f3f7c583267ef9ccc9b24c84568ed0cb47f6.tar.gz
bitbake: svn.py: Stop SVN from directly pulling from an external layer w/o fetcher
Add a new option to the svn fetcher url "externals=allowed". This will allow a user to enable svn co w/ externals. However, this does avoid the fetcher, network access and mirror systems. By default we no longer allow externals in the checkout. This ensures a deterministic download. The system does attempt to identify SVN repos that have externals enabled, and will warn the user. It is up to the user to determine if these are necessary for the recipe. They may disable the warning by adding "externals=nowarn" to the url. In the future we would like to parse this list and see if the items are already in the SRC_URI for that recipe, but with SVN being in limited use these days that extra work is likely not worth the trouble. Add test cases that generated a local SVN tree, with an external source set to github bitbake in svn format. One test case checks that externals are ignored, and one checks that they in downloaded. (Bitbake rev: bf53f07c3647e57d8452a7743a2b04bcb72c80d6) Signed-off-by: Mark Hatle <mark.hatle@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/fetch2/svn.py19
-rw-r--r--bitbake/lib/bb/tests/fetch.py80
2 files changed, 99 insertions, 0 deletions
diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py
index baeb0e7eea..59ce93160c 100644
--- a/bitbake/lib/bb/fetch2/svn.py
+++ b/bitbake/lib/bb/fetch2/svn.py
@@ -91,6 +91,13 @@ class Svn(FetchMethod):
91 svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) 91 svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
92 else: 92 else:
93 suffix = "" 93 suffix = ""
94
95 # externals may be either 'allowed' or 'nowarn', but not both. Allowed
96 # will not issue a warning, but will log to the debug buffer what has likely
97 # been downloaded by SVN.
98 if not ("externals" in ud.parm and ud.parm["externals"] == "allowed"):
99 options.append("--ignore-externals")
100
94 if ud.revision: 101 if ud.revision:
95 options.append("-r %s" % ud.revision) 102 options.append("-r %s" % ud.revision)
96 suffix = "@%s" % (ud.revision) 103 suffix = "@%s" % (ud.revision)
@@ -136,6 +143,18 @@ class Svn(FetchMethod):
136 bb.fetch2.check_network_access(d, svnfetchcmd, ud.url) 143 bb.fetch2.check_network_access(d, svnfetchcmd, ud.url)
137 runfetchcmd(svnfetchcmd, d, workdir=ud.pkgdir) 144 runfetchcmd(svnfetchcmd, d, workdir=ud.pkgdir)
138 145
146 if not ("externals" in ud.parm and ud.parm["externals"] == "nowarn"):
147 # Warn the user if this had externals (won't catch them all)
148 output = runfetchcmd("svn propget svn:externals", d, workdir=ud.moddir)
149 if output:
150 if "--ignore-externals" in svnfetchcmd.split():
151 bb.warn("%s contains svn:externals." % ud.url)
152 bb.warn("These should be added to the recipe SRC_URI as necessary.")
153 bb.warn("svn fetch has ignored externals:\n%s" % output)
154 bb.warn("To disable this warning add ';externals=nowarn' to the url.")
155 else:
156 bb.debug(1, "svn repository has externals:\n%s" % output)
157
139 scmdata = ud.parm.get("scmdata", "") 158 scmdata = ud.parm.get("scmdata", "")
140 if scmdata == "keep": 159 if scmdata == "keep":
141 tar_flags = "" 160 tar_flags = ""
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 6bdf0416d7..16f975b137 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -991,6 +991,86 @@ class FetcherNetworkTest(FetcherTest):
991 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 991 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
992 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 992 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
993 993
994class SVNTest(FetcherTest):
995 def skipIfNoSvn():
996 import shutil
997 if not shutil.which("svn"):
998 return unittest.skip("svn not installed, tests being skipped")
999
1000 if not shutil.which("svnadmin"):
1001 return unittest.skip("svnadmin not installed, tests being skipped")
1002
1003 return lambda f: f
1004
1005 @skipIfNoSvn()
1006 def setUp(self):
1007 """ Create a local repository """
1008
1009 super(SVNTest, self).setUp()
1010
1011 # Create something we can fetch
1012 src_dir = tempfile.mkdtemp(dir=self.tempdir,
1013 prefix='svnfetch_srcdir_')
1014 src_dir = os.path.abspath(src_dir)
1015 bb.process.run("echo readme > README.md", cwd=src_dir)
1016
1017 # Store it in a local SVN repository
1018 repo_dir = tempfile.mkdtemp(dir=self.tempdir,
1019 prefix='svnfetch_localrepo_')
1020 repo_dir = os.path.abspath(repo_dir)
1021 bb.process.run("svnadmin create project", cwd=repo_dir)
1022
1023 self.repo_url = "file://%s/project" % repo_dir
1024 bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url),
1025 cwd=repo_dir)
1026
1027 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
1028 # Github will emulate SVN. Use this to check if we're downloding...
1029 bb.process.run("svn propset svn:externals 'bitbake http://github.com/openembedded/bitbake' .",
1030 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
1031 bb.process.run("svn commit --non-interactive -m 'Add external'",
1032 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
1033
1034 self.src_dir = src_dir
1035 self.repo_dir = repo_dir
1036
1037 @skipIfNoSvn()
1038 def tearDown(self):
1039 os.chdir(self.origdir)
1040 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
1041 print("Not cleaning up %s. Please remove manually." % self.tempdir)
1042 else:
1043 bb.utils.prunedir(self.tempdir)
1044
1045 @skipIfNoSvn()
1046 @skipIfNoNetwork()
1047 def test_noexternal_svn(self):
1048 # Always match the rev count from setUp (currently rev 2)
1049 url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '')
1050 fetcher = bb.fetch.Fetch([url], self.d)
1051 fetcher.download()
1052 os.chdir(os.path.dirname(self.unpackdir))
1053 fetcher.unpack(self.unpackdir)
1054
1055 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
1056 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
1057 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should NOT exist")
1058 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should NOT exit")
1059
1060 @skipIfNoSvn()
1061 def test_external_svn(self):
1062 # Always match the rev count from setUp (currently rev 2)
1063 url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '')
1064 fetcher = bb.fetch.Fetch([url], self.d)
1065 fetcher.download()
1066 os.chdir(os.path.dirname(self.unpackdir))
1067 fetcher.unpack(self.unpackdir)
1068
1069 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
1070 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
1071 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should exist")
1072 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should exit")
1073
994class TrustedNetworksTest(FetcherTest): 1074class TrustedNetworksTest(FetcherTest):
995 def test_trusted_network(self): 1075 def test_trusted_network(self):
996 # Ensure trusted_network returns False when the host IS in the list. 1076 # Ensure trusted_network returns False when the host IS in the list.