summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorEnrico Scholz <enrico.scholz@sigma-chemnitz.de>2022-05-19 12:05:55 +0200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2022-06-28 12:55:26 +0100
commit6ca69f66ba70c829956746cf290e39b16a950731 (patch)
tree694c9ca97495369b620b73d95a559d28a15e603e
parentd88cbe2ec1d6a58e9348be8e5b4221321d0672b1 (diff)
downloadpoky-6ca69f66ba70c829956746cf290e39b16a950731.tar.gz
lib:npm_registry: initial checkin
Helper module to: - generate meta information from package.json content. This data has a format as provided by https://registry.npmjs.org - put this meta information and the corresponding tarball in the nodejs cache. This uses an external, nodejs version specific helper script (oe-npm-cache) shipped in oe-meta To avoid further nodejs version dependencies, future versions of this module might omit the caching completely and serve meta information and tarball by an http server. (From OE-Core rev: 6cd5886ad05fee704e8a5892bd370c360c8c3b54) Signed-off-by: Enrico Scholz <enrico.scholz@sigma-chemnitz.de> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/lib/oe/npm_registry.py169
1 files changed, 169 insertions, 0 deletions
diff --git a/meta/lib/oe/npm_registry.py b/meta/lib/oe/npm_registry.py
new file mode 100644
index 0000000000..96c0affb45
--- /dev/null
+++ b/meta/lib/oe/npm_registry.py
@@ -0,0 +1,169 @@
1import bb
2import json
3import subprocess
4
5_ALWAYS_SAFE = frozenset('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
6 'abcdefghijklmnopqrstuvwxyz'
7 '0123456789'
8 '_.-~')
9
10MISSING_OK = object()
11
12REGISTRY = "https://registry.npmjs.org"
13
14# we can not use urllib.parse here because npm expects lowercase
15# hex-chars but urllib generates uppercase ones
16def uri_quote(s, safe = '/'):
17 res = ""
18 safe_set = set(safe)
19 for c in s:
20 if c in _ALWAYS_SAFE or c in safe_set:
21 res += c
22 else:
23 res += '%%%02x' % ord(c)
24 return res
25
26class PackageJson:
27 def __init__(self, spec):
28 self.__spec = spec
29
30 @property
31 def name(self):
32 return self.__spec['name']
33
34 @property
35 def version(self):
36 return self.__spec['version']
37
38 @property
39 def empty_manifest(self):
40 return {
41 'name': self.name,
42 'description': self.__spec.get('description', ''),
43 'versions': {},
44 }
45
46 def base_filename(self):
47 return uri_quote(self.name, safe = '@')
48
49 def as_manifest_entry(self, tarball_uri):
50 res = {}
51
52 ## NOTE: 'npm install' requires more than basic meta information;
53 ## e.g. it takes 'bin' from this manifest entry but not the actual
54 ## 'package.json'
55 for (idx,dflt) in [('name', None),
56 ('description', ""),
57 ('version', None),
58 ('bin', MISSING_OK),
59 ('man', MISSING_OK),
60 ('scripts', MISSING_OK),
61 ('directories', MISSING_OK),
62 ('dependencies', MISSING_OK),
63 ('devDependencies', MISSING_OK),
64 ('optionalDependencies', MISSING_OK),
65 ('license', "unknown")]:
66 if idx in self.__spec:
67 res[idx] = self.__spec[idx]
68 elif dflt == MISSING_OK:
69 pass
70 elif dflt != None:
71 res[idx] = dflt
72 else:
73 raise Exception("%s-%s: missing key %s" % (self.name,
74 self.version,
75 idx))
76
77 res['dist'] = {
78 'tarball': tarball_uri,
79 }
80
81 return res
82
83class ManifestImpl:
84 def __init__(self, base_fname, spec):
85 self.__base = base_fname
86 self.__spec = spec
87
88 def load(self):
89 try:
90 with open(self.filename, "r") as f:
91 res = json.load(f)
92 except IOError:
93 res = self.__spec.empty_manifest
94
95 return res
96
97 def save(self, meta):
98 with open(self.filename, "w") as f:
99 json.dump(meta, f, indent = 2)
100
101 @property
102 def filename(self):
103 return self.__base + ".meta"
104
105class Manifest:
106 def __init__(self, base_fname, spec):
107 self.__base = base_fname
108 self.__spec = spec
109 self.__lockf = None
110 self.__impl = None
111
112 def __enter__(self):
113 self.__lockf = bb.utils.lockfile(self.__base + ".lock")
114 self.__impl = ManifestImpl(self.__base, self.__spec)
115 return self.__impl
116
117 def __exit__(self, exc_type, exc_val, exc_tb):
118 bb.utils.unlockfile(self.__lockf)
119
120class NpmCache:
121 def __init__(self, cache):
122 self.__cache = cache
123
124 @property
125 def path(self):
126 return self.__cache
127
128 def run(self, type, key, fname):
129 subprocess.run(['oe-npm-cache', self.__cache, type, key, fname],
130 check = True)
131
132class NpmRegistry:
133 def __init__(self, path, cache):
134 self.__path = path
135 self.__cache = NpmCache(cache + '/_cacache')
136 bb.utils.mkdirhier(self.__path)
137 bb.utils.mkdirhier(self.__cache.path)
138
139 @staticmethod
140 ## This function is critical and must match nodejs expectations
141 def _meta_uri(spec):
142 return REGISTRY + '/' + uri_quote(spec.name, safe = '@')
143
144 @staticmethod
145 ## Exact return value does not matter; just make it look like a
146 ## usual registry url
147 def _tarball_uri(spec):
148 return '%s/%s/-/%s-%s.tgz' % (REGISTRY,
149 uri_quote(spec.name, safe = '@'),
150 uri_quote(spec.name, safe = '@/'),
151 spec.version)
152
153 def add_pkg(self, tarball, pkg_json):
154 pkg_json = PackageJson(pkg_json)
155 base = os.path.join(self.__path, pkg_json.base_filename())
156
157 with Manifest(base, pkg_json) as manifest:
158 meta = manifest.load()
159 tarball_uri = self._tarball_uri(pkg_json)
160
161 meta['versions'][pkg_json.version] = pkg_json.as_manifest_entry(tarball_uri)
162
163 manifest.save(meta)
164
165 ## Cache entries are a little bit dependent on the nodejs
166 ## version; version specific cache implementation must
167 ## mitigate differences
168 self.__cache.run('meta', self._meta_uri(pkg_json), manifest.filename);
169 self.__cache.run('tgz', tarball_uri, tarball);