Skip to content

Commit dfdf3af

Browse files
maresbmaresb
authored
Merge pull request #800 from maresb/fix-url-stripping
Don't also strip port when stripping credentials from a URL
2 parents 6bf8aae + 688cae1 commit dfdf3af

File tree

7 files changed

+158
-39
lines changed

7 files changed

+158
-39
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ repos:
4747
(?x)^(
4848
^conda_lock/_vendor/.*\.pyi$
4949
| ^tests/test-local-pip/setup\.py$
50-
| ^tests/test-pip-repositories/fake-private-package-1\.0\.0/setup\.py$
50+
| ^tests/test-pip-repositories/fake-private-package.*-1\.0\.0/setup\.py$
5151
)
5252
# First exclude is due to:
5353
# conda_lock/_vendor/conda/__init__.py: error: Duplicate module named "conda_lock._vendor.conda" (also at "conda_lock/_vendor/conda.pyi")

‎conda_lock/pypi_solver.py‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -409,7 +409,7 @@ def get_requirements(
409409
# https://github.com/conda/conda-lock/blob/ac31f5ddf2951ed4819295238ccf062fb2beb33c/conda_lock/_vendor/poetry/installation/executor.py#L557
410410
else:
411411
link = chooser.choose_for(op.package)
412-
url = _get_url(link)
412+
url = _get_stripped_url(link)
413413
hash = _compute_hash(link, lock_spec_hashes.get(op.package.name))
414414
if source_repository:
415415
url = source_repository.normalize_solver_url(url)
@@ -431,10 +431,43 @@ def get_requirements(
431431
return requirements
432432

433433

434-
def _get_url(link: Link) -> str:
434+
def _get_stripped_url(link: Link) -> str:
435+
"""Get the URL for a package link, stripping credentials.
436+
437+
Basic case, do nothing:
438+
>>> _get_stripped_url(Link(url="http://example.com/path/to/file"))
439+
'http://example.com/path/to/file'
440+
441+
Strip credentials:
442+
>>> _get_stripped_url(Link(url="http://user:pass@example.com/path/to/file"))
443+
'http://example.com/path/to/file'
444+
445+
Handle a port:
446+
>>> _get_stripped_url(Link(url="http://example.com:8080/path/to/file"))
447+
'http://example.com:8080/path/to/file'
448+
449+
Strip credentials while handling a port:
450+
>>> _get_stripped_url(Link(url="http://user:pass@example.com:8080/path/to/file"))
451+
'http://example.com:8080/path/to/file'
452+
453+
General case:
454+
>>> _get_stripped_url(Link(url="https://user:pass@example.com:8080/path/to/file?query#fragment"))
455+
'https://example.com:8080/path/to/file?query'
456+
"""
435457
parsed_url = urlsplit(link.url)
436-
link.url = link.url.replace(parsed_url.netloc, str(parsed_url.hostname))
437-
return link.url_without_fragment
458+
# Reconstruct the URL with just hostname:port, no credentials
459+
clean_netloc = f"{parsed_url.hostname}"
460+
if parsed_url.port is not None:
461+
clean_netloc = f"{clean_netloc}:{parsed_url.port}"
462+
return urlunsplit(
463+
(
464+
parsed_url.scheme,
465+
clean_netloc,
466+
parsed_url.path,
467+
parsed_url.query,
468+
"", # Remove fragment
469+
)
470+
)
438471

439472

440473
def _compute_hash(link: Link, lock_spec_hash: Optional[str]) -> HashModel:

‎tests/test-pip-repositories/environment.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@ channels:
22
- conda-forge
33
pip-repositories:
44
- http://$PIP_USER:$PIP_PASSWORD@private-pypi.org/api/pypi/simple
5+
- http://$PIP_USER:$PIP_PASSWORD@private-pypi-custom-port.org:8080/api/pypi/simple
56
dependencies:
67
- python=3.11
78
- requests=2.26
89
- pip:
910
- fake-private-package==1.0.0
11+
- fake-private-package-custom-port==1.0.0
1012
platforms:
1113
- linux-64

‎tests/test-pip-repositories/fake-private-package-custom-port-1.0.0/PKG-INFO‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Metadata-Version: 2.1
2+
Name: fake-private-package-custom-port
3+
Version: 1.0.0

‎tests/test-pip-repositories/fake-private-package-custom-port-1.0.0/setup.cfg‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[egg_info]
2+
tag_build =
3+
tag_date = 0

‎tests/test-pip-repositories/fake-private-package-custom-port-1.0.0/setup.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
from setuptools import setup
2+
3+
4+
setup(
5+
name="fake-private-package-custom-port",
6+
version="1.0.0",
7+
)

‎tests/test_pip_repositories.py‎

Lines changed: 105 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import base64
22
import os
3+
import re
34
import tarfile
45

56
from io import BytesIO
@@ -39,6 +40,7 @@
3940

4041
@pytest.fixture
4142
def private_package_tar(tmp_path: Path):
43+
"""A private package to be served from the default port"""
4244
sdist_path = (
4345
clone_test_dir("test-pip-repositories", tmp_path) / "fake-private-package-1.0.0"
4446
)
@@ -49,11 +51,29 @@ def private_package_tar(tmp_path: Path):
4951
return tar_path
5052

5153

54+
@pytest.fixture
55+
def private_package_tar_custom_port(tmp_path: Path):
56+
"""A second private package to be served from a non-default port"""
57+
sdist_path = (
58+
clone_test_dir("test-pip-repositories", tmp_path)
59+
/ "fake-private-package-custom-port-1.0.0"
60+
)
61+
assert sdist_path.exists()
62+
tar_path = sdist_path / "fake-private-package-custom-port-1.0.0.tar.gz"
63+
with tarfile.open(tar_path, "w:gz") as tar:
64+
tar.add(sdist_path, arcname=os.path.basename(sdist_path))
65+
return tar_path
66+
67+
5268
@pytest.fixture(
5369
autouse=True,
5470
params=["response_url_without_credentials", "response_url_with_credentials"],
5571
)
56-
def mock_private_pypi(private_package_tar: Path, request: pytest.FixtureRequest):
72+
def mock_private_pypi( # noqa: C901
73+
private_package_tar: Path,
74+
private_package_tar_custom_port: Path,
75+
request: pytest.FixtureRequest,
76+
):
5777
with requests_mock.Mocker(real_http=True) as mocker:
5878
fixture_request = request
5979

@@ -82,11 +102,14 @@ def _make_response(
82102
response.raw.write(file_handler.read())
83103
response.raw.seek(0)
84104

85-
url = urlparse(request.url)
86105
if fixture_request.param == "response_url_with_credentials":
87106
response.url = request.url
88107
else:
89-
response.url = request.url.replace(url.netloc, url.hostname)
108+
# Strip credentials using regex, preserving port if present:
109+
# ^([^:]+://) - Capture group 1: scheme (http:// or https://)
110+
# [^@]+@ - Match and remove credentials (anything up to @)
111+
# \1 - Replace with just the captured scheme
112+
response.url = re.sub(r"^([^:]+://)[^@]+@", r"\1", request.url)
90113
response.reason = reason
91114
return response
92115

@@ -105,22 +128,64 @@ def _parse_auth(request: requests.Request) -> Tuple[str, str]:
105128

106129
@mocker._adapter.add_matcher
107130
def handle_request(request: requests.Request) -> Optional[requests.Response]:
131+
"""Intercept requests to private-pypi.org and private-pypi-custom-port.org.
132+
133+
Requests to other hosts are passed through to the real internet.
134+
135+
On private-pypi.org:80, we publish fake-private-package.
136+
137+
On private-pypi-custom-port.org:8080, we publish fake-private-package-custom-port.
138+
"""
108139
url = urlparse(request.url)
109-
if url.hostname != "private-pypi.org":
140+
if url.hostname not in ["private-pypi.org", "private-pypi-custom-port.org"]:
141+
# Bail out and use normal requests.get()
110142
return None
111143
username, password = _parse_auth(request)
112144
if username != _PRIVATE_REPO_USERNAME or password != _PRIVATE_REPO_PASSWORD:
113145
return _make_response(request, status=401, reason="Not authorized")
114146
path = url.path.rstrip("/")
115-
if path == "/api/pypi/simple":
116-
return _make_response(request, status=200, text=_PRIVATE_REPO_ROOT)
117-
if path == "/api/pypi/simple/fake-private-package":
118-
return _make_response(request, status=200, text=_PRIVATE_REPO_PACKAGE)
119-
if path == "/files/fake-private-package-1.0.0.tar.gz":
120-
return _make_response(
121-
request, status=200, file=str(private_package_tar)
122-
)
123-
return _make_response(request, status=404, reason="Not Found")
147+
if url.port:
148+
port = url.port
149+
elif url.scheme == "https":
150+
port = 443
151+
elif url.scheme == "http":
152+
port = 80
153+
else:
154+
raise ValueError(f"Unknown scheme: {url.scheme}")
155+
if url.hostname == "private-pypi.org":
156+
if port != 80:
157+
return None
158+
text = ""
159+
file = None
160+
if path == "/api/pypi/simple":
161+
text = _PRIVATE_REPO_ROOT
162+
if path == "/api/pypi/simple/fake-private-package":
163+
text = _PRIVATE_REPO_PACKAGE
164+
if path == "/files/fake-private-package-1.0.0.tar.gz":
165+
file = str(private_package_tar)
166+
if text == "" and file is None:
167+
return _make_response(request, status=404, reason="Not Found")
168+
return _make_response(request, status=200, text=text, file=file)
169+
elif url.hostname == "private-pypi-custom-port.org":
170+
if port != 8080:
171+
return None
172+
text = ""
173+
file = None
174+
if path == "/api/pypi/simple":
175+
text = _PRIVATE_REPO_ROOT.replace(
176+
"fake-private-package", "fake-private-package-custom-port"
177+
)
178+
if path == "/api/pypi/simple/fake-private-package-custom-port":
179+
text = _PRIVATE_REPO_PACKAGE.replace(
180+
"fake-private-package", "fake-private-package-custom-port"
181+
)
182+
if path == "/files/fake-private-package-custom-port-1.0.0.tar.gz":
183+
file = str(private_package_tar_custom_port)
184+
if text == "" and file is None:
185+
return _make_response(request, status=404, reason="Not Found")
186+
return _make_response(request, status=200, text=text, file=file)
187+
else:
188+
return None
124189

125190
yield
126191

@@ -157,24 +222,30 @@ def test_it_uses_pip_repositories_with_env_var_substitution(
157222
lockfile_content = lockfile_path.read_text(encoding="utf-8")
158223
packages = {package.name: package for package in lockfile.package}
159224

160-
# AND the private package is in the lockfile
161-
package = packages.get("fake-private-package")
162-
assert package, lockfile_content
163-
164-
package_url = urlparse(package.url)
165-
166-
# AND the package was sourced from the private repository
167-
assert package_url.hostname == "private-pypi.org", (
168-
"Package was fetched from incorrect host. See full lock-file:\n"
169-
+ lockfile_content
170-
)
171-
172-
# AND environment variables are occluded
173-
assert package_url.username == "$PIP_USER", (
174-
"User environment variable was not respected, See full lock-file:\n"
175-
+ lockfile_content
176-
)
177-
assert package_url.password == "$PIP_PASSWORD", (
178-
"Password environment variable was not respected, See full lock-file:\n"
179-
+ lockfile_content
180-
)
225+
# AND the private packages are in the lockfile
226+
for package_name in ["fake-private-package", "fake-private-package-custom-port"]:
227+
package = packages.get(package_name)
228+
assert package, lockfile_content
229+
230+
package_url = urlparse(package.url)
231+
232+
# AND the package was sourced from the private repository
233+
expected_hostname = (
234+
"private-pypi.org"
235+
if package_name == "fake-private-package"
236+
else "private-pypi-custom-port.org"
237+
)
238+
assert package_url.hostname == expected_hostname, (
239+
"Package was fetched from incorrect host. See full lock-file:\n"
240+
+ lockfile_content
241+
)
242+
243+
# AND environment variables are occluded
244+
assert package_url.username == "$PIP_USER", (
245+
"User environment variable was not respected, See full lock-file:\n"
246+
+ lockfile_content
247+
)
248+
assert package_url.password == "$PIP_PASSWORD", (
249+
"Password environment variable was not respected, See full lock-file:\n"
250+
+ lockfile_content
251+
)

0 commit comments

Comments
 (0)