summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMark Hatle <mark.hatle@windriver.com>2019-03-27 13:40:42 -0400
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-04-01 14:48:44 +0100
commit2030e815bb1ba932c58b0c9318f97602fdd4edfb (patch)
tree6f663fd10f059f1ba8bef331ea291ac7d68b78c8
parent24fa03eae68bef9e2146e8ec5b9aed2324cabeb2 (diff)
downloadpoky-2030e815bb1ba932c58b0c9318f97602fdd4edfb.tar.gz
bitbake: gitsm: Add need_update method to determine when we are going to a new SRCREV
If the system had previously fetched a source repository for use by gitsm, and then the SRCREV was updated and the new commit already existed, the system would not re-evaluate the submodules and update them accordingly. The cause of this issue was that need_update was being used, unmodified, from the base git fetcher. It did not have any knowledge, nor did it care if we were moving commits and needed to re-evaluate what was happening due to this switch. To fix the issue, during the download process we add all processed (by gitsm) srcrevs to the git config file, as bitbake.srcrev. This allows us to use a new need_update function that not only checks if the git commit is present, but if we have previously processed this commit to ensure all of the submodule components are also present. This approach is used, instead of iterating over the submodules in need_update to avoid a potential race condition that has affected us in the past. The need_update is called only with the parent locking. Any time we need to dive into the submodules, we need to lock, and unlock them, at each stage. This opens the possibility of errors in either the code, or unintended race conditions with rm_work. This issue was discovered by William A. Kennington III <wak@google.com>. The included test case was also written by him, and included unmodified. (Bitbake rev: 30fe86d22c239afa75168cc5eb262b880886ef8a) Signed-off-by: Mark Hatle <mark.hatle@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/fetch2/gitsm.py20
-rw-r--r--bitbake/lib/bb/tests/fetch.py19
2 files changed, 39 insertions, 0 deletions
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py
index b21fed2669..32389130bd 100644
--- a/bitbake/lib/bb/fetch2/gitsm.py
+++ b/bitbake/lib/bb/fetch2/gitsm.py
@@ -147,6 +147,23 @@ class GitSM(Git):
147 147
148 return submodules != [] 148 return submodules != []
149 149
150 def need_update(self, ud, d):
151 if Git.need_update(self, ud, d):
152 return True
153
154 try:
155 # Check for the nugget dropped by the download operation
156 known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \
157 (ud.basecmd), d, workdir=ud.clonedir)
158
159 if ud.revisions[ud.names[0]] not in known_srcrevs.split():
160 return True
161 except bb.fetch2.FetchError:
162 # No srcrev nuggets, so this is new and needs to be updated
163 return True
164
165 return False
166
150 def download(self, ud, d): 167 def download(self, ud, d):
151 def download_submodule(ud, url, module, modpath, d): 168 def download_submodule(ud, url, module, modpath, d):
152 url += ";bareclone=1;nobranch=1" 169 url += ";bareclone=1;nobranch=1"
@@ -157,6 +174,9 @@ class GitSM(Git):
157 try: 174 try:
158 newfetch = Fetch([url], d, cache=False) 175 newfetch = Fetch([url], d, cache=False)
159 newfetch.download() 176 newfetch.download()
177 # Drop a nugget to add each of the srcrevs we've fetched (used by need_update)
178 runfetchcmd("%s config --add bitbake.srcrev %s" % \
179 (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
160 except Exception as e: 180 except Exception as e:
161 logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) 181 logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
162 raise 182 raise
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 257326277a..429998b34f 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -942,6 +942,25 @@ class FetcherNetworkTest(FetcherTest):
942 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 942 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
943 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 943 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
944 944
945 def test_git_submodule_update_CLI11(self):
946 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
947 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714"
948 fetcher = bb.fetch.Fetch([url], self.d)
949 fetcher.download()
950
951 # CLI11 that pulls in a newer nlohmann-json
952 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca"
953 fetcher = bb.fetch.Fetch([url], self.d)
954 fetcher.download()
955 # Previous cwd has been deleted
956 os.chdir(os.path.dirname(self.unpackdir))
957 fetcher.unpack(self.unpackdir)
958
959 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
960 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
961 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
962 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
963
945 def test_git_submodule_aktualizr(self): 964 def test_git_submodule_aktualizr(self):
946 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" 965 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
947 fetcher = bb.fetch.Fetch([url], self.d) 966 fetcher = bb.fetch.Fetch([url], self.d)