feat: Add a global cache for uv, pre-commit and global venv (#307)

Supercedes #304 and #305 by adding a global volume that contains:
- pre-commit cache
- uv cache
- uv managed python cache
- the venvs for all the projects

This allows us to use hardlink mode which makes it even faster, and
avoid the issue of anonymous volumes in devcontainers being left
dangling. Added docs to say that there is now a global venv for the
container, which is different for how it would look without any
environment variables set

Author: coretl

.devcontainer/devcontainer.json

@@ -8,16 +8,22 @@
     "remoteEnv": {
         // Allow X11 apps to run inside the container
         "DISPLAY": "${localEnv:DISPLAY}",
+        // Put things that allow it in the persistent cache
+        "PRE_COMMIT_HOME": "/cache/pre-commit",
+        "UV_CACHE_DIR": "/cache/uv",
+        "UV_PYTHON_CACHE_DIR": "/cache/uv-python",
+        // Make a venv that is specific for this workspace path as the cache is shared
+        "UV_PROJECT_ENVIRONMENT": "/cache/venv-for${localWorkspaceFolder}",
         // Do the equivalent of "activate" the venv so we don't have to "uv run" everything
-        "VIRTUAL_ENV": "/workspaces/${localWorkspaceFolderBasename}/.venv",
-        "PATH": "/workspaces/${localWorkspaceFolderBasename}/.venv/bin:${containerEnv:PATH}"
+        "VIRTUAL_ENV": "/cache/venv-for${localWorkspaceFolder}",
+        "PATH": "/cache/venv-for${localWorkspaceFolder}/bin:${containerEnv:PATH}"
     },
     "customizations": {
         "vscode": {
             // Set *default* container specific settings.json values on container create.
             "settings": {
                 // Use the container's python by default
-                "python.defaultInterpreterPath": "/workspaces/${localWorkspaceFolderBasename}/.venv/bin/python",
+                "python.defaultInterpreterPath": "/cache/venv-for${localWorkspaceFolder}/bin/python",
                 // Don't activate the venv as it is already in the PATH
                 "python.terminal.activateEnvInCurrentTerminal": false,
                 "python.terminal.activateEnvironment": false,
@@ -38,9 +44,7 @@
         }
     },
     // Create the config folder for the bash-config feature and uv cache
-    // NOTE: The uv cache can get large, DLS users should read
-    // https://dev-guide.diamond.ac.uk/linux-user-environment/how-tos/disk-quota-troubleshooting
-    "initializeCommand": "mkdir -p ${localEnv:HOME}/.config/terminal-config ${localEnv:HOME}/.cache/uv",
+    "initializeCommand": "mkdir -p ${localEnv:HOME}/.config/terminal-config",
     "runArgs": [
         // Allow the container to access the host X11 display and EPICS CA
         "--net=host",
@@ -54,20 +58,15 @@
             "target": "/user-terminal-config",
             "type": "bind"
         },
-        // Keep a persistent cross container cache for uv
+        // Keep a persistent cross container cache for uv, pre-commit, and the venvs
         {
-            "source": "${localEnv:HOME}/.cache/uv",
-            "target": "/root/.cache/uv",
-            "type": "bind"
-        },
-        // Use a volume mount for the uv venv so it is local to the container
-        {
-            "target": "/workspaces/${localWorkspaceFolderBasename}/.venv",
+            "source": "devcontainer-shared-cache",
+            "target": "/cache",
             "type": "volume"
         }
     ],
     // Mount the parent as /workspaces so we can pip install peers as editable
     "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind",
-    // After the container is created, install the python project in editable form
-    "postCreateCommand": "uv sync && uv run pre-commit install --install-hooks"
+    // After the container is created, recreate the venv then make pre-commit first run faster
+    "postCreateCommand": "uv venv --clear && uv sync && pre-commit install --install-hooks"
 }

docs/how-to/dev-install.md

@@ -20,11 +20,11 @@ code .
 
 Click on 'Reopen in Container' when prompted on startup or, if vscode is already running, open the command menu with CTRL+SHIFT+P, search for and run 'Reopen in Container'.
 
-Open a new terminal
+The developer container creates and activates a venv (stored in `/cache/venv-for/path/to/project`) and this will be managed by any `uv sync` command as explained in [](./lock-requirements.md). Any rebuild of the container will recreate this venv, but the dependencies will be stored in a cross container cache so that rebuilds are quick.
 
 ## Build and test
 
-Now you have a development environment you can run the tests in a terminal:
+Now you have a development environment you can run the tests in a new terminal:
 
 ```
 tox -p

docs/how-to/lock-requirements.md

@@ -32,3 +32,11 @@ uv sync --upgrade
 ```{seealso}
 [The uv docs on locking and syncing](https://docs.astral.sh/uv/concepts/projects/sync)
 ```
+
+## Modifying the venv to add other projects
+
+Peer projects (those checked out next to the project) are visible in the devcontainer, and can be added into the venv by running `uv pip install -e ../other_project`. This will allow live changes made in this other project to be immediately reflected in the venv.
+
+```{note}
+This venv is activated by default, and global to the container, so if you `uv sync` from `other_project` then it will **replace** the contents of the venv with `other_project`'s dependencies.
+```