Switch to new cache format and new cache location.

Author: itamarst

news/2984.bugfix

@@ -0,0 +1 @@
+pip uses less memory when caching large packages. As a result, there is a new on-disk cache format stored in a new directory ($PIP_CACHE_DIR/http-v2).

src/pip/_internal/cli/req_command.py

@@ -123,7 +123,7 @@ def _build_session(
             ssl_context = None
 
         session = PipSession(
-            cache=os.path.join(cache_dir, "http") if cache_dir else None,
+            cache=os.path.join(cache_dir, "http-v2") if cache_dir else None,
             retries=retries if retries is not None else options.retries,
             trusted_hosts=options.trusted_hosts,
             index_urls=self._get_index_urls(options),

src/pip/_internal/commands/cache.py

@@ -93,25 +93,31 @@ def get_cache_info(self, options: Values, args: List[Any]) -> None:
         num_http_files = len(self._find_http_files(options))
         num_packages = len(self._find_wheels(options, "*"))
 
-        http_cache_location = self._cache_dir(options, "http")
+        http_cache_location = self._cache_dir(options, "http-v2")
+        old_http_cache_location = self._cache_dir(options, "http")
         wheels_cache_location = self._cache_dir(options, "wheels")
         http_cache_size = filesystem.format_directory_size(http_cache_location)
+        old_http_cache_size = filesystem.format_directory_size(old_http_cache_location)
         wheels_cache_size = filesystem.format_directory_size(wheels_cache_location)
 
         message = (
             textwrap.dedent(
                 """
-                    Package index page cache location: {http_cache_location}
-                    Package index page cache size: {http_cache_size}
-                    Number of HTTP files: {num_http_files}
+                    Package index page cache location (new): {http_cache_location}
+                    Package index page cache location (old): {old_http_cache_location}
+                    Package index page cache size (new): {http_cache_size}
+                    Package index page cache size (old): {old_http_cache_size}
+                    Number of HTTP files (old+new cache): {num_http_files}
                     Locally built wheels location: {wheels_cache_location}
                     Locally built wheels size: {wheels_cache_size}
                     Number of locally built wheels: {package_count}
                 """
             )
             .format(
                 http_cache_location=http_cache_location,
+                old_http_cache_location=old_http_cache_location,
                 http_cache_size=http_cache_size,
+                old_http_cache_size=old_http_cache_size,
                 num_http_files=num_http_files,
                 wheels_cache_location=wheels_cache_location,
                 package_count=num_packages,
@@ -195,8 +201,11 @@ def _cache_dir(self, options: Values, subdir: str) -> str:
         return os.path.join(options.cache_dir, subdir)
 
     def _find_http_files(self, options: Values) -> List[str]:
-        http_dir = self._cache_dir(options, "http")
-        return filesystem.find_files(http_dir, "*")
+        old_http_dir = self._cache_dir(options, "http")
+        new_http_dir = self._cache_dir(options, "http-v2")
+        return filesystem.find_files(old_http_dir, "*") + filesystem.find_files(
+            new_http_dir, "*"
+        )
 
     def _find_wheels(self, options: Values, pattern: str) -> List[str]:
         wheel_dir = self._cache_dir(options, "wheels")

src/pip/_internal/network/cache.py

@@ -3,10 +3,10 @@
 
 import os
 from contextlib import contextmanager
-from typing import Generator, Optional
+from typing import BinaryIO, Generator, Optional
 
-from pip._vendor.cachecontrol.cache import BaseCache
-from pip._vendor.cachecontrol.caches import FileCache
+from pip._vendor.cachecontrol.cache import SeparateBodyBaseCache
+from pip._vendor.cachecontrol.caches import SeparateBodyFileCache
 from pip._vendor.requests.models import Response
 
 from pip._internal.utils.filesystem import adjacent_tmp_file, replace
@@ -28,7 +28,7 @@ def suppressed_cache_errors() -> Generator[None, None, None]:
         pass
 
 
-class SafeFileCache(BaseCache):
+class SafeFileCache(SeparateBodyBaseCache):
     """
     A file based cache which is safe to use even when the target directory may
     not be accessible or writable.
@@ -43,7 +43,7 @@ def _get_cache_path(self, name: str) -> str:
         # From cachecontrol.caches.file_cache.FileCache._fn, brought into our
         # class for backwards-compatibility and to avoid using a non-public
         # method.
-        hashed = FileCache.encode(name)
+        hashed = SeparateBodyFileCache.encode(name)
         parts = list(hashed[:5]) + [hashed]
         return os.path.join(self.directory, *parts)
 
@@ -53,17 +53,33 @@ def get(self, key: str) -> Optional[bytes]:
             with open(path, "rb") as f:
                 return f.read()
 
-    def set(self, key: str, value: bytes, expires: Optional[int] = None) -> None:
-        path = self._get_cache_path(key)
+    def _write(self, path: str, data: bytes) -> None:
         with suppressed_cache_errors():
             ensure_dir(os.path.dirname(path))
 
             with adjacent_tmp_file(path) as f:
-                f.write(value)
+                f.write(data)
 
             replace(f.name, path)
 
+    def set(self, key: str, value: bytes, expires: Optional[int] = None) -> None:
+        path = self._get_cache_path(key)
+        self._write(path, value)
+
     def delete(self, key: str) -> None:
         path = self._get_cache_path(key)
         with suppressed_cache_errors():
             os.remove(path)
+            os.remove(path + ".body")
+
+    def get_body(self, key: str) -> Optional[BinaryIO]:
+        path = self._get_cache_path(key) + ".body"
+        with suppressed_cache_errors():
+            return open(path, "rb")
+
+    def set_body(self, key: str, body: Optional[bytes]) -> None:
+        if body is None:
+            # Workaround for https://github.com/ionrock/cachecontrol/issues/276
+            return
+        path = self._get_cache_path(key) + ".body"
+        self._write(path, body)

tests/functional/test_cache.py

@@ -20,7 +20,7 @@ def cache_dir(script: PipTestEnvironment) -> str:
 
 @pytest.fixture
 def http_cache_dir(cache_dir: str) -> str:
-    return os.path.normcase(os.path.join(cache_dir, "http"))
+    return os.path.normcase(os.path.join(cache_dir, "http-v2"))
 
 
 @pytest.fixture
@@ -211,7 +211,7 @@ def test_cache_info(
 ) -> None:
     result = script.pip("cache", "info")
 
-    assert f"Package index page cache location: {http_cache_dir}" in result.stdout
+    assert f"Package index page cache location (new): {http_cache_dir}" in result.stdout
     assert f"Locally built wheels location: {wheel_cache_dir}" in result.stdout
     num_wheels = len(wheel_cache_files)
     assert f"Number of locally built wheels: {num_wheels}" in result.stdout