|
| 1 | +From 7be9dc649cdf4a7404c145ce56e658e82a3e894a Mon Sep 17 00:00:00 2001 |
| 2 | +From: Serhiy Storchaka < [email protected]> |
| 3 | +Date: Fri, 31 Oct 2025 15:49:51 +0200 |
| 4 | +Subject: [PATCH 1/2] gh-136065: Fix quadratic complexity in |
| 5 | + os.path.expandvars() (GH-134952) (cherry picked from commit |
| 6 | + f029e8db626ddc6e3a3beea4eff511a71aaceb5c) |
| 7 | +MIME-Version: 1.0 |
| 8 | +Content-Type: text/plain; charset=UTF-8 |
| 9 | +Content-Transfer-Encoding: 8bit |
| 10 | + |
| 11 | +Co-authored-by: Serhiy Storchaka < [email protected]> |
| 12 | +Co-authored-by: Łukasz Langa < [email protected]> |
| 13 | +--- |
| 14 | + Lib/ntpath.py | 126 ++++++------------ |
| 15 | + Lib/posixpath.py | 43 +++--- |
| 16 | + Lib/test/test_genericpath.py | 14 ++ |
| 17 | + Lib/test/test_ntpath.py | 22 ++- |
| 18 | + ...-05-30-22-33-27.gh-issue-136065.bu337o.rst | 1 + |
| 19 | + 5 files changed, 93 insertions(+), 113 deletions(-) |
| 20 | + create mode 100644 Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst |
| 21 | + |
| 22 | +diff --git a/Lib/ntpath.py b/Lib/ntpath.py |
| 23 | +index 1bef630..393d358 100644 |
| 24 | +--- a/Lib/ntpath.py |
| 25 | ++++ b/Lib/ntpath.py |
| 26 | +@@ -409,17 +409,23 @@ def expanduser(path): |
| 27 | + # XXX With COMMAND.COM you can use any characters in a variable name, |
| 28 | + # XXX except '^|<>='. |
| 29 | + |
| 30 | ++_varpattern = r"'[^']*'?|%(%|[^%]*%?)|\$(\$|[-\w]+|\{[^}]*\}?)" |
| 31 | ++_varsub = None |
| 32 | ++_varsubb = None |
| 33 | ++ |
| 34 | + def expandvars(path): |
| 35 | + """Expand shell variables of the forms $var, ${var} and %var%. |
| 36 | + |
| 37 | + Unknown variables are left unchanged.""" |
| 38 | + path = os.fspath(path) |
| 39 | ++ global _varsub, _varsubb |
| 40 | + if isinstance(path, bytes): |
| 41 | + if b'$' not in path and b'%' not in path: |
| 42 | + return path |
| 43 | +- import string |
| 44 | +- varchars = bytes(string.ascii_letters + string.digits + '_-', 'ascii') |
| 45 | +- quote = b'\'' |
| 46 | ++ if not _varsubb: |
| 47 | ++ import re |
| 48 | ++ _varsubb = re.compile(_varpattern.encode(), re.ASCII).sub |
| 49 | ++ sub = _varsubb |
| 50 | + percent = b'%' |
| 51 | + brace = b'{' |
| 52 | + rbrace = b'}' |
| 53 | +@@ -428,94 +434,44 @@ def expandvars(path): |
| 54 | + else: |
| 55 | + if '$' not in path and '%' not in path: |
| 56 | + return path |
| 57 | +- import string |
| 58 | +- varchars = string.ascii_letters + string.digits + '_-' |
| 59 | +- quote = '\'' |
| 60 | ++ if not _varsub: |
| 61 | ++ import re |
| 62 | ++ _varsub = re.compile(_varpattern, re.ASCII).sub |
| 63 | ++ sub = _varsub |
| 64 | + percent = '%' |
| 65 | + brace = '{' |
| 66 | + rbrace = '}' |
| 67 | + dollar = '$' |
| 68 | + environ = os.environ |
| 69 | +- res = path[:0] |
| 70 | +- index = 0 |
| 71 | +- pathlen = len(path) |
| 72 | +- while index < pathlen: |
| 73 | +- c = path[index:index+1] |
| 74 | +- if c == quote: # no expansion within single quotes |
| 75 | +- path = path[index + 1:] |
| 76 | +- pathlen = len(path) |
| 77 | +- try: |
| 78 | +- index = path.index(c) |
| 79 | +- res += c + path[:index + 1] |
| 80 | +- except ValueError: |
| 81 | +- res += c + path |
| 82 | +- index = pathlen - 1 |
| 83 | +- elif c == percent: # variable or '%' |
| 84 | +- if path[index + 1:index + 2] == percent: |
| 85 | +- res += c |
| 86 | +- index += 1 |
| 87 | +- else: |
| 88 | +- path = path[index+1:] |
| 89 | +- pathlen = len(path) |
| 90 | +- try: |
| 91 | +- index = path.index(percent) |
| 92 | +- except ValueError: |
| 93 | +- res += percent + path |
| 94 | +- index = pathlen - 1 |
| 95 | +- else: |
| 96 | +- var = path[:index] |
| 97 | +- try: |
| 98 | +- if environ is None: |
| 99 | +- value = os.fsencode(os.environ[os.fsdecode(var)]) |
| 100 | +- else: |
| 101 | +- value = environ[var] |
| 102 | +- except KeyError: |
| 103 | +- value = percent + var + percent |
| 104 | +- res += value |
| 105 | +- elif c == dollar: # variable or '$$' |
| 106 | +- if path[index + 1:index + 2] == dollar: |
| 107 | +- res += c |
| 108 | +- index += 1 |
| 109 | +- elif path[index + 1:index + 2] == brace: |
| 110 | +- path = path[index+2:] |
| 111 | +- pathlen = len(path) |
| 112 | +- try: |
| 113 | +- index = path.index(rbrace) |
| 114 | +- except ValueError: |
| 115 | +- res += dollar + brace + path |
| 116 | +- index = pathlen - 1 |
| 117 | +- else: |
| 118 | +- var = path[:index] |
| 119 | +- try: |
| 120 | +- if environ is None: |
| 121 | +- value = os.fsencode(os.environ[os.fsdecode(var)]) |
| 122 | +- else: |
| 123 | +- value = environ[var] |
| 124 | +- except KeyError: |
| 125 | +- value = dollar + brace + var + rbrace |
| 126 | +- res += value |
| 127 | +- else: |
| 128 | +- var = path[:0] |
| 129 | +- index += 1 |
| 130 | +- c = path[index:index + 1] |
| 131 | +- while c and c in varchars: |
| 132 | +- var += c |
| 133 | +- index += 1 |
| 134 | +- c = path[index:index + 1] |
| 135 | +- try: |
| 136 | +- if environ is None: |
| 137 | +- value = os.fsencode(os.environ[os.fsdecode(var)]) |
| 138 | +- else: |
| 139 | +- value = environ[var] |
| 140 | +- except KeyError: |
| 141 | +- value = dollar + var |
| 142 | +- res += value |
| 143 | +- if c: |
| 144 | +- index -= 1 |
| 145 | ++ |
| 146 | ++ def repl(m): |
| 147 | ++ lastindex = m.lastindex |
| 148 | ++ if lastindex is None: |
| 149 | ++ return m[0] |
| 150 | ++ name = m[lastindex] |
| 151 | ++ if lastindex == 1: |
| 152 | ++ if name == percent: |
| 153 | ++ return name |
| 154 | ++ if not name.endswith(percent): |
| 155 | ++ return m[0] |
| 156 | ++ name = name[:-1] |
| 157 | + else: |
| 158 | +- res += c |
| 159 | +- index += 1 |
| 160 | +- return res |
| 161 | ++ if name == dollar: |
| 162 | ++ return name |
| 163 | ++ if name.startswith(brace): |
| 164 | ++ if not name.endswith(rbrace): |
| 165 | ++ return m[0] |
| 166 | ++ name = name[1:-1] |
| 167 | ++ |
| 168 | ++ try: |
| 169 | ++ if environ is None: |
| 170 | ++ return os.fsencode(os.environ[os.fsdecode(name)]) |
| 171 | ++ else: |
| 172 | ++ return environ[name] |
| 173 | ++ except KeyError: |
| 174 | ++ return m[0] |
| 175 | ++ |
| 176 | ++ return sub(repl, path) |
| 177 | + |
| 178 | + |
| 179 | + # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B. |
| 180 | +diff --git a/Lib/posixpath.py b/Lib/posixpath.py |
| 181 | +index 90a6f54..6306f14 100644 |
| 182 | +--- a/Lib/posixpath.py |
| 183 | ++++ b/Lib/posixpath.py |
| 184 | +@@ -314,42 +314,41 @@ def expanduser(path): |
| 185 | + # This expands the forms $variable and ${variable} only. |
| 186 | + # Non-existent variables are left unchanged. |
| 187 | + |
| 188 | +-_varprog = None |
| 189 | +-_varprogb = None |
| 190 | ++_varpattern = r'\$(\w+|\{[^}]*\}?)' |
| 191 | ++_varsub = None |
| 192 | ++_varsubb = None |
| 193 | + |
| 194 | + def expandvars(path): |
| 195 | + """Expand shell variables of form $var and ${var}. Unknown variables |
| 196 | + are left unchanged.""" |
| 197 | + path = os.fspath(path) |
| 198 | +- global _varprog, _varprogb |
| 199 | ++ global _varsub, _varsubb |
| 200 | + if isinstance(path, bytes): |
| 201 | + if b'$' not in path: |
| 202 | + return path |
| 203 | +- if not _varprogb: |
| 204 | ++ if not _varsubb: |
| 205 | + import re |
| 206 | +- _varprogb = re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII) |
| 207 | +- search = _varprogb.search |
| 208 | ++ _varsubb = re.compile(_varpattern.encode(), re.ASCII).sub |
| 209 | ++ sub = _varsubb |
| 210 | + start = b'{' |
| 211 | + end = b'}' |
| 212 | + environ = getattr(os, 'environb', None) |
| 213 | + else: |
| 214 | + if '$' not in path: |
| 215 | + return path |
| 216 | +- if not _varprog: |
| 217 | ++ if not _varsub: |
| 218 | + import re |
| 219 | +- _varprog = re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII) |
| 220 | +- search = _varprog.search |
| 221 | ++ _varsub = re.compile(_varpattern, re.ASCII).sub |
| 222 | ++ sub = _varsub |
| 223 | + start = '{' |
| 224 | + end = '}' |
| 225 | + environ = os.environ |
| 226 | +- i = 0 |
| 227 | +- while True: |
| 228 | +- m = search(path, i) |
| 229 | +- if not m: |
| 230 | +- break |
| 231 | +- i, j = m.span(0) |
| 232 | +- name = m.group(1) |
| 233 | +- if name.startswith(start) and name.endswith(end): |
| 234 | ++ |
| 235 | ++ def repl(m): |
| 236 | ++ name = m[1] |
| 237 | ++ if name.startswith(start): |
| 238 | ++ if not name.endswith(end): |
| 239 | ++ return m[0] |
| 240 | + name = name[1:-1] |
| 241 | + try: |
| 242 | + if environ is None: |
| 243 | +@@ -357,13 +356,11 @@ def expandvars(path): |
| 244 | + else: |
| 245 | + value = environ[name] |
| 246 | + except KeyError: |
| 247 | +- i = j |
| 248 | ++ return m[0] |
| 249 | + else: |
| 250 | +- tail = path[j:] |
| 251 | +- path = path[:i] + value |
| 252 | +- i = len(path) |
| 253 | +- path += tail |
| 254 | +- return path |
| 255 | ++ return value |
| 256 | ++ |
| 257 | ++ return sub(repl, path) |
| 258 | + |
| 259 | + |
| 260 | + # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B. |
| 261 | +diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py |
| 262 | +index 3eefb72..1cec587 100644 |
| 263 | +--- a/Lib/test/test_genericpath.py |
| 264 | ++++ b/Lib/test/test_genericpath.py |
| 265 | +@@ -7,6 +7,7 @@ import os |
| 266 | + import sys |
| 267 | + import unittest |
| 268 | + import warnings |
| 269 | ++from test import support |
| 270 | + from test.support import is_emscripten |
| 271 | + from test.support import os_helper |
| 272 | + from test.support import warnings_helper |
| 273 | +@@ -443,6 +444,19 @@ class CommonTest(GenericTest): |
| 274 | + os.fsencode('$bar%s bar' % nonascii)) |
| 275 | + check(b'$spam}bar', os.fsencode('%s}bar' % nonascii)) |
| 276 | + |
| 277 | ++ @support.requires_resource('cpu') |
| 278 | ++ def test_expandvars_large(self): |
| 279 | ++ expandvars = self.pathmodule.expandvars |
| 280 | ++ with os_helper.EnvironmentVarGuard() as env: |
| 281 | ++ env.clear() |
| 282 | ++ env["A"] = "B" |
| 283 | ++ n = 100_000 |
| 284 | ++ self.assertEqual(expandvars('$A'*n), 'B'*n) |
| 285 | ++ self.assertEqual(expandvars('${A}'*n), 'B'*n) |
| 286 | ++ self.assertEqual(expandvars('$A!'*n), 'B!'*n) |
| 287 | ++ self.assertEqual(expandvars('${A}A'*n), 'BA'*n) |
| 288 | ++ self.assertEqual(expandvars('${'*10*n), '${'*10*n) |
| 289 | ++ |
| 290 | + def test_abspath(self): |
| 291 | + self.assertIn("foo", self.pathmodule.abspath("foo")) |
| 292 | + with warnings.catch_warnings(): |
| 293 | +diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py |
| 294 | +index 93d7011..1cbcefc 100644 |
| 295 | +--- a/Lib/test/test_ntpath.py |
| 296 | ++++ b/Lib/test/test_ntpath.py |
| 297 | +@@ -7,8 +7,7 @@ import sys |
| 298 | + import unittest |
| 299 | + import warnings |
| 300 | + from ntpath import ALLOW_MISSING |
| 301 | +-from test.support import cpython_only, os_helper |
| 302 | +-from test.support import TestFailed, is_emscripten |
| 303 | ++from test.support import os_helper, is_emscripten |
| 304 | + from test.support.os_helper import FakePath |
| 305 | + from test import test_genericpath |
| 306 | + from tempfile import TemporaryFile |
| 307 | +@@ -58,7 +57,7 @@ def tester(fn, wantResult): |
| 308 | + fn = fn.replace("\\", "\\\\") |
| 309 | + gotResult = eval(fn) |
| 310 | + if wantResult != gotResult and _norm(wantResult) != _norm(gotResult): |
| 311 | +- raise TestFailed("%s should return: %s but returned: %s" \ |
| 312 | ++ raise support.TestFailed("%s should return: %s but returned: %s" \ |
| 313 | + %(str(fn), str(wantResult), str(gotResult))) |
| 314 | + |
| 315 | + # then with bytes |
| 316 | +@@ -74,7 +73,7 @@ def tester(fn, wantResult): |
| 317 | + warnings.simplefilter("ignore", DeprecationWarning) |
| 318 | + gotResult = eval(fn) |
| 319 | + if _norm(wantResult) != _norm(gotResult): |
| 320 | +- raise TestFailed("%s should return: %s but returned: %s" \ |
| 321 | ++ raise support.TestFailed("%s should return: %s but returned: %s" \ |
| 322 | + %(str(fn), str(wantResult), repr(gotResult))) |
| 323 | + |
| 324 | + |
| 325 | +@@ -927,6 +926,19 @@ class TestNtpath(NtpathTestCase): |
| 326 | + check('%spam%bar', '%sbar' % nonascii) |
| 327 | + check('%{}%bar'.format(nonascii), 'ham%sbar' % nonascii) |
| 328 | + |
| 329 | ++ @support.requires_resource('cpu') |
| 330 | ++ def test_expandvars_large(self): |
| 331 | ++ expandvars = ntpath.expandvars |
| 332 | ++ with os_helper.EnvironmentVarGuard() as env: |
| 333 | ++ env.clear() |
| 334 | ++ env["A"] = "B" |
| 335 | ++ n = 100_000 |
| 336 | ++ self.assertEqual(expandvars('%A%'*n), 'B'*n) |
| 337 | ++ self.assertEqual(expandvars('%A%A'*n), 'BA'*n) |
| 338 | ++ self.assertEqual(expandvars("''"*n + '%%'), "''"*n + '%') |
| 339 | ++ self.assertEqual(expandvars("%%"*n), "%"*n) |
| 340 | ++ self.assertEqual(expandvars("$$"*n), "$"*n) |
| 341 | ++ |
| 342 | + def test_expanduser(self): |
| 343 | + tester('ntpath.expanduser("test")', 'test') |
| 344 | + |
| 345 | +@@ -1252,7 +1264,7 @@ class TestNtpath(NtpathTestCase): |
| 346 | + self.assertTrue(os.path.exists(r"\\.\CON")) |
| 347 | + |
| 348 | + @unittest.skipIf(sys.platform != 'win32', "Fast paths are only for win32") |
| 349 | +- @cpython_only |
| 350 | ++ @support.cpython_only |
| 351 | + def test_fast_paths_in_use(self): |
| 352 | + # There are fast paths of these functions implemented in posixmodule.c. |
| 353 | + # Confirm that they are being used, and not the Python fallbacks in |
| 354 | +diff --git a/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst b/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst |
| 355 | +new file mode 100644 |
| 356 | +index 0000000..1d152bb |
| 357 | +--- /dev/null |
| 358 | ++++ b/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst |
| 359 | +@@ -0,0 +1 @@ |
| 360 | ++Fix quadratic complexity in :func:`os.path.expandvars`. |
| 361 | +-- |
| 362 | +2.45.4 |
| 363 | + |
| 364 | + |
| 365 | +From 9207fe3c613099c07d6a2500305d214b466ff842 Mon Sep 17 00:00:00 2001 |
| 366 | +From: =?UTF-8?q?=C5=81ukasz=20Langa?= < [email protected]> |
| 367 | +Date: Fri, 31 Oct 2025 17:32:52 +0100 |
| 368 | +Subject: [PATCH 2/2] Oops |
| 369 | + |
| 370 | +Signed-off-by: Azure Linux Security Servicing Account < [email protected]> |
| 371 | +Upstream-reference: https://github.com/python/cpython/pull/140847.patch |
| 372 | +--- |
| 373 | + Lib/test/test_ntpath.py | 1 + |
| 374 | + 1 file changed, 1 insertion(+) |
| 375 | + |
| 376 | +diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py |
| 377 | +index 1cbcefc..310b452 100644 |
| 378 | +--- a/Lib/test/test_ntpath.py |
| 379 | ++++ b/Lib/test/test_ntpath.py |
| 380 | +@@ -7,6 +7,7 @@ import sys |
| 381 | + import unittest |
| 382 | + import warnings |
| 383 | + from ntpath import ALLOW_MISSING |
| 384 | ++from test import support |
| 385 | + from test.support import os_helper, is_emscripten |
| 386 | + from test.support.os_helper import FakePath |
| 387 | + from test import test_genericpath |
| 388 | +-- |
| 389 | +2.45.4 |
| 390 | + |
0 commit comments