Remove credential filling (#11)

Author: connesy

README.md

@@ -105,7 +105,7 @@ $ venv install requirements.txt
 ```
 are equivalent.
 
-The installed packages are then _locked_ into the corresponding `.lock`-file, e.g. running `venv install dev-requirements.txt` will lock those installed packages into `dev-requirements.lock`.
+The installed packages are then _locked_ into the corresponding `.lock`-file, e.g. running `venv install dev-requirements.txt` will lock those installed packages into `dev-requirements.lock`[^1].
 
 Installing packages this way makes sure that they are tracked, since installing them with `pip install` will keep no record of which packages have been installed in the environment, making it difficult to reproduce later on.
 
@@ -126,7 +126,7 @@ matplotlib
 The `-r requirements.txt` will make sure that installing development requirements also install production requirements.
 
 ## Reproducing environment
-To install a reproducible environment, you need to install from a `.lock`-file, since those have all versions of all requirements locked. From a clean environment (no packages installed yet), run
+To install a reproducible environment, you need to install from a `.lock`-file, since those have all versions of all requirements locked[^1]. From a clean environment (no packages installed yet), run
 ```console
 $ venv install requirements.lock
 ```
@@ -185,3 +185,6 @@ Releases are made by creating a branch `release/vX.X.X` from `develop`, where `X
 ## License
 
 [MIT](https://choosealicense.com/licenses/mit/)
+
+[^1]: A current limitation of using `pip freeze` under the hood is that installing packages from a version control system (VCS) URL that requires authentication, e.g. `private_package @ git+https://USERNAME:[email protected]/my-user/private-package`, the authentication is not locked (see https://github.com/pypa/pip/issues/12365).
+These credentials can either be inserted manually into the generated `.lock`-file, or the credentials can instead be stored in a `.netrc` file, which `pip install` will then reference when running `pip install`: https://pip.pypa.io/en/stable/topics/authentication/#netrc-support

src/venv-cli/completions/bash/venv_completion.sh

@@ -39,12 +39,18 @@ _venv() {
             COMPREPLY+=( ${python_versions[*]} )
             COMPREPLY+=( ${help_options[*]} )
             ;;
-        "install"|"lock")
+        "install")
             # Generate completions for requirement and lock file paths
             COMPREPLY+=( $(compgen -f -X '!(*.txt|*.lock)' -- "${cur_word}" | sort) )
             COMPREPLY+=( ${help_options[*]} )
             compopt -o plusdirs +o nosort  # Add directories after generated completions
             ;;
+        "lock")
+            # Generate completions for lock file paths
+            COMPREPLY+=( $(compgen -f -X '!(*.lock)' -- "${cur_word}" | sort) )
+            COMPREPLY+=( ${help_options[*]} )
+            compopt -o plusdirs +o nosort  # Add directories after generated completions
+            ;;
         "sync")
             # Generate completions for lock file paths
             COMPREPLY+=( $(compgen -f -X '!*.lock' -- "${cur_word}" | sort) )
@@ -62,12 +68,6 @@ _venv() {
             # Nothing to generate
             ;;
     esac
-
-    # Special case for 'venv lock requirements.txt <TAB>', where only *.lock files should be suggested
-    if [ "${COMP_WORDS[COMP_CWORD-2]}" == "lock" ] && [[ "${prev_word}" =~ ^.*\.txt$ ]]; then
-        COMPREPLY+=( $(compgen -f -X '!*.lock' -- "${cur_word}" | sort) )
-        compopt -o plusdirs +o nosort  # Add directories after generated completions
-    fi
 }
 
 complete -F _venv venv

src/venv-cli/venv.sh

@@ -227,105 +227,41 @@ venv::install() {
 }
 
 
-venv::_fill_credentials() {
-  ### Read VCS URL auth from requirements file and fill in lock file URLs
-  local requirements_file="$1"
-  local lock_file="$2"
-
-  # Loop over every line in requirements file
-  while IFS= read -r req_line; do
-    # Extract package name from requirement
-    local package=$(echo "${req_line}" | command awk '{print $1}')
-    # Extract VCS URL from requirement
-    local url_req=$(echo "${req_line}" | command awk '{print $3}')
-
-    # Extract environment variable(s) from the URL
-    local env_vars=$(echo "${url_req}" | command grep -oE "${_env_var_auth_pattern}")
-    if [ -z "${env_vars}" ]; then
-      # No env vars in the url (or no url at all), skip line
-      continue
-    fi
-
-    # Use sed to fill in $env_vars after "https://"-section, so a line like
-    # 'asdf-lib @ git+https://github.com/someuser/asdf-lib@commithash'
-    # becomes e.g.
-    # 'asdf-lib @ git+https://${AUTH_TOKEN}@github.com/someuser/asdf-lib@commithash'
-    # or
-    # 'asdf-lib @ git+https://${USERNAME}:${PASSWORD}@github.com/someuser/asdf-lib@commithash'
-    local before_auth_pattern="^(${package} @ [a-zA-Z+]*?https?://)"
-    local after_auth_pattern="(.*?)$"
-    local fill_pattern="\1${env_vars}@\2"
-    command sed -i -E "s|${before_auth_pattern}${after_auth_pattern}|${fill_pattern}|" "${lock_file}"
-
-  done < "${requirements_file}"
-}
-
-
 venv::lock() {
   if venv::_check_if_help_requested "$1"; then
     echo "venv lock [<lock file>|<lock file prefix>]"
-    echo "venv lock <requirements file> <lock file>"
     echo
     echo "Lock all installed package versions and write them to <lock file>."
     echo "The <lock file> must be in the form '*requirements.lock'."
     echo
     echo "If <lock file prefix> is specified instead, locks the requirements to"
-    echo "a file called '<lock file prefix>-requirements.lock'."
+    echo "a file called '<lock file prefix>-requirements.lock', e.g."
+    echo "'venv lock dev' locks requirements to 'dev-requirements.lock'."
     echo
     echo "If no <lock file> is specified, defaults to 'requirements.lock'."
     echo
-    echo "This function uses 'pip freeze' to lock the requirements, but since"
-    echo "'pip freeze' does not include the auth-part of VCS URLs, this command"
-    echo "needs a reference 'requirements.txt'-file to extract the credentials from."
-    echo
-    echo "In the first form, where only <lock file> is specified, this command"
-    echo "looks for a reference <requirements file> with the same stem as <lock file>,"
-    echo "and with a '.txt' extension, e.g. 'venv lock dev-requirements.lock' will look"
-    echo "for a reference file 'dev-requirements.txt'."
-    echo
-    echo "In the second form, both the reference <requirements file> and the <lock file>"
-    echo "are specified."
-    echo
     echo "Examples:"
     echo "$ venv lock"
-    echo "This will lock requirements into 'requirements.lock',"
-    echo "referencing 'requirements.txt'."
+    echo "This will lock requirements into 'requirements.lock'."
     echo
     echo "$ venv lock dev-requirements.lock"
-    echo "This will lock requirements into 'dev-requirements.lock',"
-    echo "referencing 'dev-requirements.txt'."
+    echo "This will lock requirements into 'dev-requirements.lock'."
     echo
-    echo "$ venv lock dev"
-    echo "This will lock requirements into 'dev-requirements.lock',"
-    echo "referencing 'dev-requirements.txt'."
+    echo "$ venv lock ci"
+    echo "This will lock requirements into 'ci-requirements.lock'."
     return "${_success}"
   fi
 
-  local requirements_file
   local lock_file
   if [ -z "$1" ]; then
-    # If nothing was passed, default to "requirements.lock" with "requirements.txt"
-    # as reference
+    # If nothing was passed, default to "requirements.lock"
     lock_file="requirements.lock"
-    requirements_file="requirements.txt"
 
   elif [[ "$1" = *"."* ]]; then
     # If first argument looks like a file name ...
-
     if venv::_check_lock_requirements_file "$1" -q; then
-      # In this case, the first argument is a lock file
+      # ... and is a lock file
       lock_file="$1"
-      requirements_file="$(venv::_get_requirements_from_lock "$1")"
-      shift
-
-    elif $(venv::_check_install_requirements_file "$1" -q \
-        && venv::_check_lock_requirements_file "$2" -q); then
-      # In this case, the first argument is a requirements file and the second
-      # argument is a lock file
-      requirements_file="$1"
-      lock_file="$2"
-      shift 2
-
     else
       venv::raise "Input file(s) had wrong format. See 'venv lock --help' for more info."
       return "$?"
@@ -334,23 +270,11 @@ venv::lock() {
   else
     # If first argument is not a full filename, assume it is a lock file prefix
     lock_file="$1-requirements.lock"
-    requirements_file="$1-requirements.txt"
-  fi
-
-  if [ ! -f "${requirements_file}" ]; then
-    venv::raise "No reference requirements file found with name '${requirements_file}', aborting."
-    return "$?"
   fi
 
   # Write locked requirements into lock file
   pip freeze --require-virtualenv > "${lock_file}"
 
-  # Since 'pip freeze' does not include the auth-part of VCS URLs, we have to
-  # get those from the reference requirements file
-  if ! venv::_fill_credentials "${requirements_file}" "${lock_file}"; then
-    return "${_fail}"
-  fi
-
   venv::color_echo "${_green}" "Locked requirements in ${lock_file}"
 }
 

tests/helpers.py

@@ -1,7 +1,11 @@
 import os
 import subprocess
 import sys
+from functools import wraps
 from pathlib import Path
+from typing import Callable
+
+from tests.types import P, RawFilesDict, RequirementsBase, RequirementsDict
 
 RequirementFiles = dict[str, Path]
 current_python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
@@ -31,3 +35,20 @@ def run_command(commands: str | list[str], cwd: Path = Path.cwd(), activated: bo
 
     result = subprocess.run(all_commands, cwd=cwd)
     result.check_returncode()
+
+
+def collect_requirements(
+    func: Callable[P, tuple[RawFilesDict, RequirementsBase]]
+) -> Callable[P, tuple[RequirementsDict, RequirementsBase]]:
+    @wraps(func)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> tuple[RequirementsDict, RequirementsBase]:
+        files_dict, requirements_base = func(*args, **kwargs)
+        requirements_dict = {filename: "\n".join(requirements) for filename, requirements in files_dict.items()}
+        return requirements_dict, requirements_base
+
+    return wrapper
+
+
+def write_files(files: RequirementsDict, dir: Path) -> None:
+    for filename, contents in files.items():
+        (dir / filename).write_text(contents + "\n")