From f2d5967982359b924406e3937a8e32807a5bb533 Mon Sep 17 00:00:00 2001 From: Nathan Rossi Date: Fri, 2 Feb 2018 23:51:15 +1000 Subject: bitbake: fetch2: Handle missing donestamp file when content is valid In order to allow users to manually populate the download directory with valid content change the assumption that missing the donestamp file means unfetched content. This allows users to populate the download dir without needing to create dummy .done files such that a user does not need a PREMIRROR when using BB_NO_NETWORK to provide valid content files in the download directory. To ensure the correct result this change also fails first if the localpath does not exist. This prevents further parts of the function attempting to calculating the checksum on non-existent files. This also fixes some edge conditions around where if the donestamp exists but the localpath does not it returns, and did not remove the donestamp. Also added test cases to cover this use case and additional use cases where for example the fetcher does not support checksums. (Bitbake rev: a335dbbb65d5b56e71d98cf3e4fa9bfbec1dcde6) Signed-off-by: Nathan Rossi Signed-off-by: Richard Purdie --- bitbake/lib/bb/fetch2/__init__.py | 21 ++++---- bitbake/lib/bb/tests/fetch.py | 104 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 11 deletions(-) (limited to 'bitbake') diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index 6bd040493e..72d6092deb 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py @@ -643,26 +643,25 @@ def verify_donestamp(ud, d, origud=None): if not ud.needdonestamp or (origud and not origud.needdonestamp): return True - if not os.path.exists(ud.donestamp): + if not os.path.exists(ud.localpath): + # local path does not exist + if os.path.exists(ud.donestamp): + # done stamp exists, but the downloaded file does not; the done stamp + # must be incorrect, re-trigger the download + bb.utils.remove(ud.donestamp) return False if (not ud.method.supports_checksum(ud) or (origud and not origud.method.supports_checksum(origud))): - # done stamp exists, checksums not supported; assume the local file is - # current - return True - - if not os.path.exists(ud.localpath): - # done stamp exists, but the downloaded file does not; the done stamp - # must be incorrect, re-trigger the download - bb.utils.remove(ud.donestamp) - return False + # if done stamp exists and checksums not supported; assume the local + # file is current + return os.path.exists(ud.donestamp) precomputed_checksums = {} # Only re-use the precomputed checksums if the donestamp is newer than the # file. Do not rely on the mtime of directories, though. If ud.localpath is # a directory, there will probably not be any checksums anyway. - if (os.path.isdir(ud.localpath) or + if os.path.exists(ud.donestamp) and (os.path.isdir(ud.localpath) or os.path.getmtime(ud.localpath) < os.path.getmtime(ud.donestamp)): try: with open(ud.donestamp, "rb") as cachefile: diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py index 62b88f1380..d711c77d35 100644 --- a/bitbake/lib/bb/tests/fetch.py +++ b/bitbake/lib/bb/tests/fetch.py @@ -20,6 +20,7 @@ # import unittest +import hashlib import tempfile import subprocess import collections @@ -522,6 +523,109 @@ class FetcherLocalTest(FetcherTest): with self.assertRaises(bb.fetch2.UnpackError): self.fetchUnpack(['file://a;subdir=/bin/sh']) +class FetcherNoNetworkTest(FetcherTest): + def setUp(self): + super().setUp() + # all test cases are based on not having network + self.d.setVar("BB_NO_NETWORK", "1") + + def test_missing(self): + string = "this is a test file\n".encode("utf-8") + self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) + self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) + + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + + def test_valid_missing_donestamp(self): + # create the file in the download directory with correct hash + string = "this is a test file\n".encode("utf-8") + with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f: + f.write(string) + + self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) + self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) + + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) + fetcher.download() + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + + def test_invalid_missing_donestamp(self): + # create an invalid file in the download directory with incorrect hash + string = "this is a test file\n".encode("utf-8") + with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): + pass + + self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) + self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) + + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + # the existing file should not exist or should have be moved to "bad-checksum" + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + + def test_nochecksums_missing(self): + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + # ssh fetch does not support checksums + fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) + # attempts to download with missing donestamp + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + + def test_nochecksums_missing_donestamp(self): + # create a file in the download directory + with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): + pass + + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + # ssh fetch does not support checksums + fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) + # attempts to download with missing donestamp + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + + def test_nochecksums_has_donestamp(self): + # create a file in the download directory with the donestamp + with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): + pass + with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): + pass + + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + # ssh fetch does not support checksums + fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) + # should not fetch + fetcher.download() + # both files should still exist + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + + def test_nochecksums_missing_has_donestamp(self): + # create a file in the download directory with the donestamp + with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): + pass + + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + # ssh fetch does not support checksums + fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + # both files should still exist + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_fetch(self): -- cgit v1.2.3-54-g00ecf