Bugfix: path normalization applied to URLs

The `-r` and `-c` path normalization logic did not check if the
requested requirements or constraint files were remote files being
loaded, e.g., via HTTPS. As a result, a usage like
`-c https://example.com/constraints.txt` incorrectly got converted to
a path and normalized, stripping one of the two slashes which
separates the scheme from the netloc.

A regression test is added which reproduces the bug, and the gap in
detection is closed by checking explicitly if inputs can parse as
URIs. Any given URI (including file URIs) will be exempted from the
path rewrites.

The check is implemented by calling `urllib.parse.urlparse()` and
checking the `scheme` attribute of the parse result.

Author: sirosen

changelog.d/2223.bugfix.md

@@ -0,0 +1,2 @@
+Fixed a bug which removed slashes from URLs in ``-r`` and ``-c`` in the output
+of ``pip-compile`` -- by {user}`sirosen`.

piptools/_compat/pip_compat.py

@@ -2,6 +2,7 @@
 
 import optparse
 import pathlib
+import urllib.parse
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Callable, Iterable, Iterator, Set, cast
 
@@ -140,6 +141,10 @@ def _relativize_comes_from_location(original_comes_from: str, /) -> str:
     # split on the space
     prefix, space_sep, suffix = original_comes_from.partition(" ")
 
+    # if the value part is a URI, return the original
+    if _is_uri(suffix):
+        return original_comes_from
+
     file_path = pathlib.Path(suffix)
 
     # if the path was not absolute, normalize to posix-style and finish processing
@@ -169,11 +174,34 @@ def _normalize_comes_from_location(original_comes_from: str, /) -> str:
     # split on the space
     prefix, space_sep, suffix = original_comes_from.partition(" ")
 
+    # if the value part is a URI, return the original
+    if _is_uri(suffix):
+        return original_comes_from
+
     # convert to a posix-style path
     suffix = pathlib.Path(suffix).as_posix()
     return f"{prefix}{space_sep}{suffix}"
 
 
+def _is_uri(value: str) -> bool:
+    """
+    Test a string to see if it is a URI.
+
+    The test is performed by trying a URL parse and seeing is a scheme is populated.
+
+    This means that according to this rule, valid URLs such as
+    ``example.com/data/my_pip_constraints.txt`` may fail to count as URIs.
+    However, we cannot safely distinguish such strings from real filesystem paths.
+    e.g., ``./example.com/`` may be a directory.
+
+    Importantly, realistic usage such as
+    ``-c https://example.com/constraints.txt``
+    is properly detected by this technique.
+    """
+    parse_result = urllib.parse.urlparse(value)
+    return parse_result.scheme != ""
+
+
 def create_wheel_cache(cache_dir: str, format_control: str | None = None) -> WheelCache:
     kwargs: dict[str, str | None] = {"cache_dir": cache_dir}
     if PIP_VERSION[:2] <= (23, 0):

tests/test_cli_compile.py

@@ -14,6 +14,7 @@
 from unittest.mock import MagicMock
 
 import pytest
+from pip._internal.network.session import PipSession
 from pip._internal.req.constructors import install_req_from_line
 from pip._internal.utils.hashes import FAVORITE_HASH
 from pip._internal.utils.urls import path_to_url
@@ -4044,3 +4045,78 @@ def test_second_order_requirements_relative_path_in_separate_dir(
             # via -r {output_path}
         """
     )
+
+
+@pytest.mark.parametrize(
+    "input_path_absolute", (True, False), ids=("absolute-input", "relative-input")
+)
+def test_url_constraints_are_not_treated_as_file_paths(
+    pip_conf,
+    make_package,
+    runner,
+    tmp_path,
+    monkeypatch,
+    input_path_absolute,
+):
+    """
+    Test normalization of ``-c`` constraints when the constraints are HTTPS URLs.
+    The constraints should be preserved verbatim.
+
+    This is a regression test for
+    https://github.com/jazzband/pip-tools/issues/2223
+    """
+    reqs_in = tmp_path / "requirements.in"
+    constraints_url = "https://example.com/files/common_constraints.txt"
+    reqs_in.write_text(
+        f"""
+        small-fake-a
+        -c {constraints_url}
+        """
+    )
+
+    input_dir_path = tmp_path if input_path_absolute else pathlib.Path(".")
+    input_path = (input_dir_path / "requirements.in").as_posix()
+
+    # TODO: find a better way of mocking the callout to get the constraints
+    #       file (use `responses`?)
+    #
+    # we need a mock response for `GET https://...` as fetched by pip
+    # although this is fragile, it can be adapted if pip changes
+    def fake_url_get(url):
+        response = mock.Mock()
+        response.reason = "Ok"
+        response.status_code = 200
+        response.url = url
+        response.text = "small-fake-a==0.2"
+        return response
+
+    mock_get = mock.Mock(side_effect=fake_url_get)
+
+    with monkeypatch.context() as revertable_ctx:
+        revertable_ctx.chdir(tmp_path)
+        with mock.patch.object(PipSession, "get", mock_get):
+            out = runner.invoke(
+                cli,
+                [
+                    "--output-file",
+                    "-",
+                    "--quiet",
+                    "--no-header",
+                    "--no-emit-options",
+                    "-r",
+                    input_path,
+                ],
+            )
+
+    # sanity check, pip should have tried to fetch the constraints
+    mock_get.assert_called_once_with(constraints_url)
+
+    assert out.exit_code == 0
+    assert out.stdout == dedent(
+        f"""\
+        small-fake-a==0.2
+            # via
+            #   -c {constraints_url}
+            #   -r {input_path}
+        """
+    )