From f6728edb7e022b27223d28bd80ce40c2f2374a13 Mon Sep 17 00:00:00 2001 From: Jean-Marie LEMETAYER Date: Fri, 24 Jan 2020 18:08:14 +0100 Subject: bitbake: tests/fetch: add npmsw tests This commit adds some tests to validate the npmsw fetcher: - bb.tests.fetch.NPMTest.test_npmsw - bb.tests.fetch.NPMTest.test_npmsw_bad_checksum - bb.tests.fetch.NPMTest.test_npmsw_destsuffix - bb.tests.fetch.NPMTest.test_npmsw_dev - bb.tests.fetch.NPMTest.test_npmsw_mirrors - bb.tests.fetch.NPMTest.test_npmsw_no_network_no_tarball - bb.tests.fetch.NPMTest.test_npmsw_no_network_with_tarball - bb.tests.fetch.NPMTest.test_npmsw_npm_reusability - bb.tests.fetch.NPMTest.test_npmsw_premirrors (Bitbake rev: ba205df20b6a07a4b1125332601c6c54c7b019b5) Signed-off-by: Jean-Marie LEMETAYER Signed-off-by: Richard Purdie --- bitbake/lib/bb/tests/fetch.py | 251 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 251 insertions(+) (limited to 'bitbake/lib') diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py index 5eeb64c512..45dc9e5d08 100644 --- a/bitbake/lib/bb/tests/fetch.py +++ b/bitbake/lib/bb/tests/fetch.py @@ -2220,3 +2220,254 @@ class NPMTest(FetcherTest): url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example' with self.assertRaises(bb.fetch2.MissingParameterError): fetcher = bb.fetch.Fetch([url], self.d) + + def create_shrinkwrap_file(self, data): + import json + datadir = os.path.join(self.tempdir, 'data') + swfile = os.path.join(datadir, 'npm-shrinkwrap.json') + bb.utils.mkdirhier(datadir) + with open(swfile, 'w') as f: + json.dump(data, f) + # Also configure the S directory + self.sdir = os.path.join(self.unpackdir, 'S') + self.d.setVar('S', self.sdir) + return swfile + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw(self): + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=', + 'dependencies': { + 'content-type': { + 'version': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', + 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', + 'dependencies': { + 'cookie': { + 'version': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', + 'from': 'git+https://github.com/jshttp/cookie.git' + } + } + } + } + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) + fetcher.unpack(self.unpackdir) + self.assertTrue(os.path.exists(os.path.join(self.sdir, 'npm-shrinkwrap.json'))) + self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json'))) + self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json'))) + self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json'))) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw_dev(self): + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + }, + 'content-type': { + 'version': '1.0.4', + 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', + 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', + 'dev': True + } + } + }) + # Fetch with dev disabled + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) + self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) + # Fetch with dev enabled + fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d) + fetcher.download() + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw_destsuffix(self): + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json'))) + + def test_npmsw_no_network_no_tarball(self): + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + } + } + }) + self.d.setVar('BB_NO_NETWORK', '1') + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw_no_network_with_tarball(self): + # Fetch once to get a tarball + fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) + fetcher.download() + # Disable network access + self.d.setVar('BB_NO_NETWORK', '1') + # Fetch again + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json'))) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw_npm_reusability(self): + # Fetch once with npmsw + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + # Disable network access + self.d.setVar('BB_NO_NETWORK', '1') + # Fetch again with npm + fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json'))) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw_bad_checksum(self): + # Try to fetch with bad checksum + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg=' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + with self.assertRaises(bb.fetch2.FetchError): + fetcher.download() + # Fetch correctly to get a tarball + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz') + self.assertTrue(os.path.exists(localpath)) + # Modify the tarball + bad = b'bad checksum' + with open(localpath, 'wb') as f: + f.write(bad) + # Verify that the tarball is fetched again + fetcher.download() + badsum = hashlib.sha1(bad).hexdigest() + self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum)) + self.assertTrue(os.path.exists(localpath)) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw_premirrors(self): + # Fetch once to get a tarball + fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) + ud = fetcher.ud[fetcher.urls[0]] + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) + # Setup the mirror + mirrordir = os.path.join(self.tempdir, 'mirror') + bb.utils.mkdirhier(mirrordir) + os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) + self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir) + self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') + # Fetch again + self.assertFalse(os.path.exists(ud.localpath)) + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw_mirrors(self): + # Fetch once to get a tarball + fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) + ud = fetcher.ud[fetcher.urls[0]] + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) + # Setup the mirror + mirrordir = os.path.join(self.tempdir, 'mirror') + bb.utils.mkdirhier(mirrordir) + os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) + self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir) + # Fetch again with invalid url + self.assertFalse(os.path.exists(ud.localpath)) + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://invalid', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) -- cgit v1.2.3-54-g00ecf