diff options
Diffstat (limited to 'meta/recipes-devtools/python/python3')
-rw-r--r-- | meta/recipes-devtools/python/python3/CVE-2021-23336.patch | 548 |
1 files changed, 548 insertions, 0 deletions
diff --git a/meta/recipes-devtools/python/python3/CVE-2021-23336.patch b/meta/recipes-devtools/python/python3/CVE-2021-23336.patch new file mode 100644 index 0000000000..27893f69fb --- /dev/null +++ b/meta/recipes-devtools/python/python3/CVE-2021-23336.patch | |||
@@ -0,0 +1,548 @@ | |||
1 | From e3110c3cfbb7daa690d54d0eff6c264c870a71bf Mon Sep 17 00:00:00 2001 | ||
2 | From: Senthil Kumaran <senthil@uthcode.com> | ||
3 | Date: Mon, 15 Feb 2021 10:15:02 -0800 | ||
4 | Subject: [PATCH] [3.8] bpo-42967: only use '&' as a query string separator | ||
5 | (GH-24297) (#24529) | ||
6 | MIME-Version: 1.0 | ||
7 | Content-Type: text/plain; charset=UTF-8 | ||
8 | Content-Transfer-Encoding: 8bit | ||
9 | |||
10 | * bpo-42967: only use '&' as a query string separator (#24297) | ||
11 | |||
12 | bpo-42967: [security] Address a web cache-poisoning issue reported in | ||
13 | urllib.parse.parse_qsl(). | ||
14 | |||
15 | urllib.parse will only us "&" as query string separator by default | ||
16 | instead of both ";" and "&" as allowed in earlier versions. An optional | ||
17 | argument seperator with default value "&" is added to specify the | ||
18 | separator. | ||
19 | |||
20 | Co-authored-by: Éric Araujo <merwok@netwok.org> | ||
21 | Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> | ||
22 | Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> | ||
23 | Co-authored-by: Éric Araujo <merwok@netwok.org> | ||
24 | (cherry picked from commit fcbe0cb04d35189401c0c880ebfb4311e952d776) | ||
25 | |||
26 | * [3.8] bpo-42967: only use '&' as a query string separator (GH-24297) | ||
27 | |||
28 | bpo-42967: [security] Address a web cache-poisoning issue reported in urllib.parse.parse_qsl(). | ||
29 | |||
30 | urllib.parse will only us "&" as query string separator by default instead of both ";" and "&" as allowed in earlier versions. An optional argument seperator with default value "&" is added to specify the separator. | ||
31 | |||
32 | Co-authored-by: Éric Araujo <merwok@netwok.org> | ||
33 | Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> | ||
34 | Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> | ||
35 | Co-authored-by: Éric Araujo <merwok@netwok.org>. | ||
36 | (cherry picked from commit fcbe0cb04d35189401c0c880ebfb4311e952d776) | ||
37 | |||
38 | Co-authored-by: Adam Goldschmidt <adamgold7@gmail.com> | ||
39 | |||
40 | * Update correct version information. | ||
41 | |||
42 | * fix docs and make logic clearer | ||
43 | |||
44 | Co-authored-by: Adam Goldschmidt <adamgold7@gmail.com> | ||
45 | Co-authored-by: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> | ||
46 | |||
47 | Upstream-Status: Backport [https://github.com/python/cpython/commit/e3110c3cfbb7daa690d54d0eff6c264c870a71bf] | ||
48 | CVE: CVE-2020-23336 | ||
49 | Signed-off-by: Chee Yang Lee <chee.yang.lee@intel.com> | ||
50 | |||
51 | --- | ||
52 | Doc/library/cgi.rst | 11 ++- | ||
53 | Doc/library/urllib.parse.rst | 22 +++++- | ||
54 | Doc/whatsnew/3.6.rst | 13 ++++ | ||
55 | Doc/whatsnew/3.7.rst | 13 ++++ | ||
56 | Doc/whatsnew/3.8.rst | 13 ++++ | ||
57 | Lib/cgi.py | 23 ++++--- | ||
58 | Lib/test/test_cgi.py | 29 ++++++-- | ||
59 | Lib/test/test_urlparse.py | 68 +++++++++++++------ | ||
60 | Lib/urllib/parse.py | 19 ++++-- | ||
61 | .../2021-02-14-15-59-16.bpo-42967.YApqDS.rst | 1 + | ||
62 | 10 files changed, 166 insertions(+), 46 deletions(-) | ||
63 | create mode 100644 Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst | ||
64 | |||
65 | diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst | ||
66 | index 4048592e7361f..880074bed6026 100644 | ||
67 | --- a/Doc/library/cgi.rst | ||
68 | +++ b/Doc/library/cgi.rst | ||
69 | @@ -277,14 +277,16 @@ These are useful if you want more control, or if you want to employ some of the | ||
70 | algorithms implemented in this module in other circumstances. | ||
71 | |||
72 | |||
73 | -.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False) | ||
74 | +.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False, separator="&") | ||
75 | |||
76 | Parse a query in the environment or from a file (the file defaults to | ||
77 | - ``sys.stdin``). The *keep_blank_values* and *strict_parsing* parameters are | ||
78 | + ``sys.stdin``). The *keep_blank_values*, *strict_parsing* and *separator* parameters are | ||
79 | passed to :func:`urllib.parse.parse_qs` unchanged. | ||
80 | |||
81 | + .. versionchanged:: 3.8.8 | ||
82 | + Added the *separator* parameter. | ||
83 | |||
84 | -.. function:: parse_multipart(fp, pdict, encoding="utf-8", errors="replace") | ||
85 | +.. function:: parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator="&") | ||
86 | |||
87 | Parse input of type :mimetype:`multipart/form-data` (for file uploads). | ||
88 | Arguments are *fp* for the input file, *pdict* for a dictionary containing | ||
89 | @@ -303,6 +305,9 @@ algorithms implemented in this module in other circumstances. | ||
90 | Added the *encoding* and *errors* parameters. For non-file fields, the | ||
91 | value is now a list of strings, not bytes. | ||
92 | |||
93 | + .. versionchanged:: 3.8.8 | ||
94 | + Added the *separator* parameter. | ||
95 | + | ||
96 | |||
97 | .. function:: parse_header(string) | ||
98 | |||
99 | diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst | ||
100 | index 25e5cc1a6ce0b..fcad7076e6c77 100644 | ||
101 | --- a/Doc/library/urllib.parse.rst | ||
102 | +++ b/Doc/library/urllib.parse.rst | ||
103 | @@ -165,7 +165,7 @@ or on combining URL components into a URL string. | ||
104 | now raise :exc:`ValueError`. | ||
105 | |||
106 | |||
107 | -.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None) | ||
108 | +.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') | ||
109 | |||
110 | Parse a query string given as a string argument (data of type | ||
111 | :mimetype:`application/x-www-form-urlencoded`). Data are returned as a | ||
112 | @@ -190,6 +190,9 @@ or on combining URL components into a URL string. | ||
113 | read. If set, then throws a :exc:`ValueError` if there are more than | ||
114 | *max_num_fields* fields read. | ||
115 | |||
116 | + The optional argument *separator* is the symbol to use for separating the | ||
117 | + query arguments. It defaults to ``&``. | ||
118 | + | ||
119 | Use the :func:`urllib.parse.urlencode` function (with the ``doseq`` | ||
120 | parameter set to ``True``) to convert such dictionaries into query | ||
121 | strings. | ||
122 | @@ -201,8 +204,14 @@ or on combining URL components into a URL string. | ||
123 | .. versionchanged:: 3.8 | ||
124 | Added *max_num_fields* parameter. | ||
125 | |||
126 | + .. versionchanged:: 3.8.8 | ||
127 | + Added *separator* parameter with the default value of ``&``. Python | ||
128 | + versions earlier than Python 3.8.8 allowed using both ``;`` and ``&`` as | ||
129 | + query parameter separator. This has been changed to allow only a single | ||
130 | + separator key, with ``&`` as the default separator. | ||
131 | + | ||
132 | |||
133 | -.. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None) | ||
134 | +.. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') | ||
135 | |||
136 | Parse a query string given as a string argument (data of type | ||
137 | :mimetype:`application/x-www-form-urlencoded`). Data are returned as a list of | ||
138 | @@ -226,6 +235,9 @@ or on combining URL components into a URL string. | ||
139 | read. If set, then throws a :exc:`ValueError` if there are more than | ||
140 | *max_num_fields* fields read. | ||
141 | |||
142 | + The optional argument *separator* is the symbol to use for separating the | ||
143 | + query arguments. It defaults to ``&``. | ||
144 | + | ||
145 | Use the :func:`urllib.parse.urlencode` function to convert such lists of pairs into | ||
146 | query strings. | ||
147 | |||
148 | @@ -235,6 +247,12 @@ or on combining URL components into a URL string. | ||
149 | .. versionchanged:: 3.8 | ||
150 | Added *max_num_fields* parameter. | ||
151 | |||
152 | + .. versionchanged:: 3.8.8 | ||
153 | + Added *separator* parameter with the default value of ``&``. Python | ||
154 | + versions earlier than Python 3.8.8 allowed using both ``;`` and ``&`` as | ||
155 | + query parameter separator. This has been changed to allow only a single | ||
156 | + separator key, with ``&`` as the default separator. | ||
157 | + | ||
158 | |||
159 | .. function:: urlunparse(parts) | ||
160 | |||
161 | diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst | ||
162 | index 85a6657fdfbda..03a877a3d9178 100644 | ||
163 | --- a/Doc/whatsnew/3.6.rst | ||
164 | +++ b/Doc/whatsnew/3.6.rst | ||
165 | @@ -2443,3 +2443,16 @@ because of the behavior of the socket option ``SO_REUSEADDR`` in UDP. For more | ||
166 | details, see the documentation for ``loop.create_datagram_endpoint()``. | ||
167 | (Contributed by Kyle Stanley, Antoine Pitrou, and Yury Selivanov in | ||
168 | :issue:`37228`.) | ||
169 | + | ||
170 | +Notable changes in Python 3.6.13 | ||
171 | +================================ | ||
172 | + | ||
173 | +Earlier Python versions allowed using both ``;`` and ``&`` as | ||
174 | +query parameter separators in :func:`urllib.parse.parse_qs` and | ||
175 | +:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with | ||
176 | +newer W3C recommendations, this has been changed to allow only a single | ||
177 | +separator key, with ``&`` as the default. This change also affects | ||
178 | +:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected | ||
179 | +functions internally. For more details, please see their respective | ||
180 | +documentation. | ||
181 | +(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) | ||
182 | diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst | ||
183 | index 4933cba3990b1..824dc13e0c6fd 100644 | ||
184 | --- a/Doc/whatsnew/3.7.rst | ||
185 | +++ b/Doc/whatsnew/3.7.rst | ||
186 | @@ -2556,3 +2556,16 @@ because of the behavior of the socket option ``SO_REUSEADDR`` in UDP. For more | ||
187 | details, see the documentation for ``loop.create_datagram_endpoint()``. | ||
188 | (Contributed by Kyle Stanley, Antoine Pitrou, and Yury Selivanov in | ||
189 | :issue:`37228`.) | ||
190 | + | ||
191 | +Notable changes in Python 3.7.10 | ||
192 | +================================ | ||
193 | + | ||
194 | +Earlier Python versions allowed using both ``;`` and ``&`` as | ||
195 | +query parameter separators in :func:`urllib.parse.parse_qs` and | ||
196 | +:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with | ||
197 | +newer W3C recommendations, this has been changed to allow only a single | ||
198 | +separator key, with ``&`` as the default. This change also affects | ||
199 | +:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected | ||
200 | +functions internally. For more details, please see their respective | ||
201 | +documentation. | ||
202 | +(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) | ||
203 | diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst | ||
204 | index 1a192800b2f02..632ccc1f2c40a 100644 | ||
205 | --- a/Doc/whatsnew/3.8.rst | ||
206 | +++ b/Doc/whatsnew/3.8.rst | ||
207 | @@ -2251,3 +2251,16 @@ The constant values of future flags in the :mod:`__future__` module | ||
208 | are updated in order to prevent collision with compiler flags. Previously | ||
209 | ``PyCF_ALLOW_TOP_LEVEL_AWAIT`` was clashing with ``CO_FUTURE_DIVISION``. | ||
210 | (Contributed by Batuhan Taskaya in :issue:`39562`) | ||
211 | + | ||
212 | +Notable changes in Python 3.8.8 | ||
213 | +=============================== | ||
214 | + | ||
215 | +Earlier Python versions allowed using both ``;`` and ``&`` as | ||
216 | +query parameter separators in :func:`urllib.parse.parse_qs` and | ||
217 | +:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with | ||
218 | +newer W3C recommendations, this has been changed to allow only a single | ||
219 | +separator key, with ``&`` as the default. This change also affects | ||
220 | +:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected | ||
221 | +functions internally. For more details, please see their respective | ||
222 | +documentation. | ||
223 | +(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) | ||
224 | diff --git a/Lib/cgi.py b/Lib/cgi.py | ||
225 | index 77ab703cc0360..1e880e51848af 100755 | ||
226 | --- a/Lib/cgi.py | ||
227 | +++ b/Lib/cgi.py | ||
228 | @@ -115,7 +115,8 @@ def closelog(): | ||
229 | # 0 ==> unlimited input | ||
230 | maxlen = 0 | ||
231 | |||
232 | -def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): | ||
233 | +def parse(fp=None, environ=os.environ, keep_blank_values=0, | ||
234 | + strict_parsing=0, separator='&'): | ||
235 | """Parse a query in the environment or from a file (default stdin) | ||
236 | |||
237 | Arguments, all optional: | ||
238 | @@ -134,6 +135,9 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): | ||
239 | strict_parsing: flag indicating what to do with parsing errors. | ||
240 | If false (the default), errors are silently ignored. | ||
241 | If true, errors raise a ValueError exception. | ||
242 | + | ||
243 | + separator: str. The symbol to use for separating the query arguments. | ||
244 | + Defaults to &. | ||
245 | """ | ||
246 | if fp is None: | ||
247 | fp = sys.stdin | ||
248 | @@ -154,7 +158,7 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): | ||
249 | if environ['REQUEST_METHOD'] == 'POST': | ||
250 | ctype, pdict = parse_header(environ['CONTENT_TYPE']) | ||
251 | if ctype == 'multipart/form-data': | ||
252 | - return parse_multipart(fp, pdict) | ||
253 | + return parse_multipart(fp, pdict, separator=separator) | ||
254 | elif ctype == 'application/x-www-form-urlencoded': | ||
255 | clength = int(environ['CONTENT_LENGTH']) | ||
256 | if maxlen and clength > maxlen: | ||
257 | @@ -178,10 +182,10 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): | ||
258 | qs = "" | ||
259 | environ['QUERY_STRING'] = qs # XXX Shouldn't, really | ||
260 | return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing, | ||
261 | - encoding=encoding) | ||
262 | + encoding=encoding, separator=separator) | ||
263 | |||
264 | |||
265 | -def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"): | ||
266 | +def parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator='&'): | ||
267 | """Parse multipart input. | ||
268 | |||
269 | Arguments: | ||
270 | @@ -205,7 +209,7 @@ def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"): | ||
271 | except KeyError: | ||
272 | pass | ||
273 | fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors, | ||
274 | - environ={'REQUEST_METHOD': 'POST'}) | ||
275 | + environ={'REQUEST_METHOD': 'POST'}, separator=separator) | ||
276 | return {k: fs.getlist(k) for k in fs} | ||
277 | |||
278 | def _parseparam(s): | ||
279 | @@ -315,7 +319,7 @@ class FieldStorage: | ||
280 | def __init__(self, fp=None, headers=None, outerboundary=b'', | ||
281 | environ=os.environ, keep_blank_values=0, strict_parsing=0, | ||
282 | limit=None, encoding='utf-8', errors='replace', | ||
283 | - max_num_fields=None): | ||
284 | + max_num_fields=None, separator='&'): | ||
285 | """Constructor. Read multipart/* until last part. | ||
286 | |||
287 | Arguments, all optional: | ||
288 | @@ -363,6 +367,7 @@ def __init__(self, fp=None, headers=None, outerboundary=b'', | ||
289 | self.keep_blank_values = keep_blank_values | ||
290 | self.strict_parsing = strict_parsing | ||
291 | self.max_num_fields = max_num_fields | ||
292 | + self.separator = separator | ||
293 | if 'REQUEST_METHOD' in environ: | ||
294 | method = environ['REQUEST_METHOD'].upper() | ||
295 | self.qs_on_post = None | ||
296 | @@ -589,7 +594,7 @@ def read_urlencoded(self): | ||
297 | query = urllib.parse.parse_qsl( | ||
298 | qs, self.keep_blank_values, self.strict_parsing, | ||
299 | encoding=self.encoding, errors=self.errors, | ||
300 | - max_num_fields=self.max_num_fields) | ||
301 | + max_num_fields=self.max_num_fields, separator=self.separator) | ||
302 | self.list = [MiniFieldStorage(key, value) for key, value in query] | ||
303 | self.skip_lines() | ||
304 | |||
305 | @@ -605,7 +610,7 @@ def read_multi(self, environ, keep_blank_values, strict_parsing): | ||
306 | query = urllib.parse.parse_qsl( | ||
307 | self.qs_on_post, self.keep_blank_values, self.strict_parsing, | ||
308 | encoding=self.encoding, errors=self.errors, | ||
309 | - max_num_fields=self.max_num_fields) | ||
310 | + max_num_fields=self.max_num_fields, separator=self.separator) | ||
311 | self.list.extend(MiniFieldStorage(key, value) for key, value in query) | ||
312 | |||
313 | klass = self.FieldStorageClass or self.__class__ | ||
314 | @@ -649,7 +654,7 @@ def read_multi(self, environ, keep_blank_values, strict_parsing): | ||
315 | else self.limit - self.bytes_read | ||
316 | part = klass(self.fp, headers, ib, environ, keep_blank_values, | ||
317 | strict_parsing, limit, | ||
318 | - self.encoding, self.errors, max_num_fields) | ||
319 | + self.encoding, self.errors, max_num_fields, self.separator) | ||
320 | |||
321 | if max_num_fields is not None: | ||
322 | max_num_fields -= 1 | ||
323 | diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py | ||
324 | index 101942de947fb..4e1506a6468b9 100644 | ||
325 | --- a/Lib/test/test_cgi.py | ||
326 | +++ b/Lib/test/test_cgi.py | ||
327 | @@ -53,12 +53,9 @@ def do_test(buf, method): | ||
328 | ("", ValueError("bad query field: ''")), | ||
329 | ("&", ValueError("bad query field: ''")), | ||
330 | ("&&", ValueError("bad query field: ''")), | ||
331 | - (";", ValueError("bad query field: ''")), | ||
332 | - (";&;", ValueError("bad query field: ''")), | ||
333 | # Should the next few really be valid? | ||
334 | ("=", {}), | ||
335 | ("=&=", {}), | ||
336 | - ("=;=", {}), | ||
337 | # This rest seem to make sense | ||
338 | ("=a", {'': ['a']}), | ||
339 | ("&=a", ValueError("bad query field: ''")), | ||
340 | @@ -73,8 +70,6 @@ def do_test(buf, method): | ||
341 | ("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}), | ||
342 | ("a=a+b&a=b+a", {'a': ['a b', 'b a']}), | ||
343 | ("x=1&y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), | ||
344 | - ("x=1;y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), | ||
345 | - ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), | ||
346 | ("Hbc5161168c542333633315dee1182227:key_store_seqid=400006&cuyer=r&view=bustomer&order_id=0bb2e248638833d48cb7fed300000f1b&expire=964546263&lobale=en-US&kid=130003.300038&ss=env", | ||
347 | {'Hbc5161168c542333633315dee1182227:key_store_seqid': ['400006'], | ||
348 | 'cuyer': ['r'], | ||
349 | @@ -201,6 +196,30 @@ def test_strict(self): | ||
350 | else: | ||
351 | self.assertEqual(fs.getvalue(key), expect_val[0]) | ||
352 | |||
353 | + def test_separator(self): | ||
354 | + parse_semicolon = [ | ||
355 | + ("x=1;y=2.0", {'x': ['1'], 'y': ['2.0']}), | ||
356 | + ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), | ||
357 | + (";", ValueError("bad query field: ''")), | ||
358 | + (";;", ValueError("bad query field: ''")), | ||
359 | + ("=;a", ValueError("bad query field: 'a'")), | ||
360 | + (";b=a", ValueError("bad query field: ''")), | ||
361 | + ("b;=a", ValueError("bad query field: 'b'")), | ||
362 | + ("a=a+b;b=b+c", {'a': ['a b'], 'b': ['b c']}), | ||
363 | + ("a=a+b;a=b+a", {'a': ['a b', 'b a']}), | ||
364 | + ] | ||
365 | + for orig, expect in parse_semicolon: | ||
366 | + env = {'QUERY_STRING': orig} | ||
367 | + fs = cgi.FieldStorage(separator=';', environ=env) | ||
368 | + if isinstance(expect, dict): | ||
369 | + for key in expect.keys(): | ||
370 | + expect_val = expect[key] | ||
371 | + self.assertIn(key, fs) | ||
372 | + if len(expect_val) > 1: | ||
373 | + self.assertEqual(fs.getvalue(key), expect_val) | ||
374 | + else: | ||
375 | + self.assertEqual(fs.getvalue(key), expect_val[0]) | ||
376 | + | ||
377 | def test_log(self): | ||
378 | cgi.log("Testing") | ||
379 | |||
380 | diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py | ||
381 | index 4ae6ed33858ce..90c8d6922629e 100644 | ||
382 | --- a/Lib/test/test_urlparse.py | ||
383 | +++ b/Lib/test/test_urlparse.py | ||
384 | @@ -32,16 +32,10 @@ | ||
385 | (b"&a=b", [(b'a', b'b')]), | ||
386 | (b"a=a+b&b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), | ||
387 | (b"a=1&a=2", [(b'a', b'1'), (b'a', b'2')]), | ||
388 | - (";", []), | ||
389 | - (";;", []), | ||
390 | - (";a=b", [('a', 'b')]), | ||
391 | - ("a=a+b;b=b+c", [('a', 'a b'), ('b', 'b c')]), | ||
392 | - ("a=1;a=2", [('a', '1'), ('a', '2')]), | ||
393 | - (b";", []), | ||
394 | - (b";;", []), | ||
395 | - (b";a=b", [(b'a', b'b')]), | ||
396 | - (b"a=a+b;b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), | ||
397 | - (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]), | ||
398 | + (";a=b", [(';a', 'b')]), | ||
399 | + ("a=a+b;b=b+c", [('a', 'a b;b=b c')]), | ||
400 | + (b";a=b", [(b';a', b'b')]), | ||
401 | + (b"a=a+b;b=b+c", [(b'a', b'a b;b=b c')]), | ||
402 | ] | ||
403 | |||
404 | # Each parse_qs testcase is a two-tuple that contains | ||
405 | @@ -68,16 +62,10 @@ | ||
406 | (b"&a=b", {b'a': [b'b']}), | ||
407 | (b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), | ||
408 | (b"a=1&a=2", {b'a': [b'1', b'2']}), | ||
409 | - (";", {}), | ||
410 | - (";;", {}), | ||
411 | - (";a=b", {'a': ['b']}), | ||
412 | - ("a=a+b;b=b+c", {'a': ['a b'], 'b': ['b c']}), | ||
413 | - ("a=1;a=2", {'a': ['1', '2']}), | ||
414 | - (b";", {}), | ||
415 | - (b";;", {}), | ||
416 | - (b";a=b", {b'a': [b'b']}), | ||
417 | - (b"a=a+b;b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), | ||
418 | - (b"a=1;a=2", {b'a': [b'1', b'2']}), | ||
419 | + (";a=b", {';a': ['b']}), | ||
420 | + ("a=a+b;b=b+c", {'a': ['a b;b=b c']}), | ||
421 | + (b";a=b", {b';a': [b'b']}), | ||
422 | + (b"a=a+b;b=b+c", {b'a':[ b'a b;b=b c']}), | ||
423 | ] | ||
424 | |||
425 | class UrlParseTestCase(unittest.TestCase): | ||
426 | @@ -884,10 +872,46 @@ def test_parse_qsl_encoding(self): | ||
427 | def test_parse_qsl_max_num_fields(self): | ||
428 | with self.assertRaises(ValueError): | ||
429 | urllib.parse.parse_qs('&'.join(['a=a']*11), max_num_fields=10) | ||
430 | - with self.assertRaises(ValueError): | ||
431 | - urllib.parse.parse_qs(';'.join(['a=a']*11), max_num_fields=10) | ||
432 | urllib.parse.parse_qs('&'.join(['a=a']*10), max_num_fields=10) | ||
433 | |||
434 | + def test_parse_qs_separator(self): | ||
435 | + parse_qs_semicolon_cases = [ | ||
436 | + (";", {}), | ||
437 | + (";;", {}), | ||
438 | + (";a=b", {'a': ['b']}), | ||
439 | + ("a=a+b;b=b+c", {'a': ['a b'], 'b': ['b c']}), | ||
440 | + ("a=1;a=2", {'a': ['1', '2']}), | ||
441 | + (b";", {}), | ||
442 | + (b";;", {}), | ||
443 | + (b";a=b", {b'a': [b'b']}), | ||
444 | + (b"a=a+b;b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), | ||
445 | + (b"a=1;a=2", {b'a': [b'1', b'2']}), | ||
446 | + ] | ||
447 | + for orig, expect in parse_qs_semicolon_cases: | ||
448 | + with self.subTest(f"Original: {orig!r}, Expected: {expect!r}"): | ||
449 | + result = urllib.parse.parse_qs(orig, separator=';') | ||
450 | + self.assertEqual(result, expect, "Error parsing %r" % orig) | ||
451 | + | ||
452 | + | ||
453 | + def test_parse_qsl_separator(self): | ||
454 | + parse_qsl_semicolon_cases = [ | ||
455 | + (";", []), | ||
456 | + (";;", []), | ||
457 | + (";a=b", [('a', 'b')]), | ||
458 | + ("a=a+b;b=b+c", [('a', 'a b'), ('b', 'b c')]), | ||
459 | + ("a=1;a=2", [('a', '1'), ('a', '2')]), | ||
460 | + (b";", []), | ||
461 | + (b";;", []), | ||
462 | + (b";a=b", [(b'a', b'b')]), | ||
463 | + (b"a=a+b;b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), | ||
464 | + (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]), | ||
465 | + ] | ||
466 | + for orig, expect in parse_qsl_semicolon_cases: | ||
467 | + with self.subTest(f"Original: {orig!r}, Expected: {expect!r}"): | ||
468 | + result = urllib.parse.parse_qsl(orig, separator=';') | ||
469 | + self.assertEqual(result, expect, "Error parsing %r" % orig) | ||
470 | + | ||
471 | + | ||
472 | def test_urlencode_sequences(self): | ||
473 | # Other tests incidentally urlencode things; test non-covered cases: | ||
474 | # Sequence and object values. | ||
475 | diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py | ||
476 | index 95be7181133b4..0c1c94f5fc986 100644 | ||
477 | --- a/Lib/urllib/parse.py | ||
478 | +++ b/Lib/urllib/parse.py | ||
479 | @@ -650,7 +650,7 @@ def unquote(string, encoding='utf-8', errors='replace'): | ||
480 | |||
481 | |||
482 | def parse_qs(qs, keep_blank_values=False, strict_parsing=False, | ||
483 | - encoding='utf-8', errors='replace', max_num_fields=None): | ||
484 | + encoding='utf-8', errors='replace', max_num_fields=None, separator='&'): | ||
485 | """Parse a query given as a string argument. | ||
486 | |||
487 | Arguments: | ||
488 | @@ -674,12 +674,15 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, | ||
489 | max_num_fields: int. If set, then throws a ValueError if there | ||
490 | are more than n fields read by parse_qsl(). | ||
491 | |||
492 | + separator: str. The symbol to use for separating the query arguments. | ||
493 | + Defaults to &. | ||
494 | + | ||
495 | Returns a dictionary. | ||
496 | """ | ||
497 | parsed_result = {} | ||
498 | pairs = parse_qsl(qs, keep_blank_values, strict_parsing, | ||
499 | encoding=encoding, errors=errors, | ||
500 | - max_num_fields=max_num_fields) | ||
501 | + max_num_fields=max_num_fields, separator=separator) | ||
502 | for name, value in pairs: | ||
503 | if name in parsed_result: | ||
504 | parsed_result[name].append(value) | ||
505 | @@ -689,7 +692,7 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, | ||
506 | |||
507 | |||
508 | def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, | ||
509 | - encoding='utf-8', errors='replace', max_num_fields=None): | ||
510 | + encoding='utf-8', errors='replace', max_num_fields=None, separator='&'): | ||
511 | """Parse a query given as a string argument. | ||
512 | |||
513 | Arguments: | ||
514 | @@ -712,19 +715,25 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, | ||
515 | max_num_fields: int. If set, then throws a ValueError | ||
516 | if there are more than n fields read by parse_qsl(). | ||
517 | |||
518 | + separator: str. The symbol to use for separating the query arguments. | ||
519 | + Defaults to &. | ||
520 | + | ||
521 | Returns a list, as G-d intended. | ||
522 | """ | ||
523 | qs, _coerce_result = _coerce_args(qs) | ||
524 | |||
525 | + if not separator or (not isinstance(separator, (str, bytes))): | ||
526 | + raise ValueError("Separator must be of type string or bytes.") | ||
527 | + | ||
528 | # If max_num_fields is defined then check that the number of fields | ||
529 | # is less than max_num_fields. This prevents a memory exhaustion DOS | ||
530 | # attack via post bodies with many fields. | ||
531 | if max_num_fields is not None: | ||
532 | - num_fields = 1 + qs.count('&') + qs.count(';') | ||
533 | + num_fields = 1 + qs.count(separator) | ||
534 | if max_num_fields < num_fields: | ||
535 | raise ValueError('Max number of fields exceeded') | ||
536 | |||
537 | - pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] | ||
538 | + pairs = [s1 for s1 in qs.split(separator)] | ||
539 | r = [] | ||
540 | for name_value in pairs: | ||
541 | if not name_value and not strict_parsing: | ||
542 | diff --git a/Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst b/Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst | ||
543 | new file mode 100644 | ||
544 | index 0000000000000..f08489b41494e | ||
545 | --- /dev/null | ||
546 | +++ b/Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst | ||
547 | @@ -0,0 +1 @@ | ||
548 | +Fix web cache poisoning vulnerability by defaulting the query args separator to ``&``, and allowing the user to choose a custom separator. | ||