Installs from private pypi with validator versioning

Author: AlejandroEsquivel

guardrails/cli/hub/utils.py

@@ -3,17 +3,24 @@
 import re
 import subprocess
 import sys
+
+from typing import Literal
+import logging
+
 from email.parser import BytesHeaderParser
-from typing import List, Literal, Union
+from typing import List, Union
 from pydash.strings import snake_case
 
 from guardrails_hub_types import Manifest
-from guardrails.cli.logger import logger
-
 
 json_format: Literal["json"] = "json"
 string_format: Literal["string"] = "string"
 
+logger = logging.getLogger(__name__)
+
+json_format = "json"
+string_format = "string"
+
 
 def pip_process(
     action: str,
@@ -34,47 +41,50 @@ def pip_process(
         env = dict(os.environ)
         if no_color:
             env["NO_COLOR"] = "true"
-        if not quiet:
-            logger.debug(f"decoding output from pip {action} {package}")
-            output = subprocess.check_output(command, env=env)
-        else:
-            output = subprocess.check_output(
-                command, stderr=subprocess.DEVNULL, env=env
-            )
+
+        result = subprocess.run(
+            command,
+            env=env,
+            capture_output=True,  # Capture both stdout and stderr
+            text=True,  # Automatically decode to strings
+            check=True,  # Automatically raise error on non-zero exit code
+        )
 
         if format == json_format:
-            parsed = BytesHeaderParser().parsebytes(output)
             try:
                 remove_color_codes = re.compile(r"\x1b\[[0-9;]*m")
-                parsed_as_string = re.sub(
-                    remove_color_codes, "", parsed.as_string().strip()
-                )
+                parsed_as_string = re.sub(remove_color_codes, "", result.stdout.strip())
                 return json.loads(parsed_as_string)
             except Exception:
                 logger.debug(
-                    f"JSON parse exception in decoding output from pip \
-{action} {package}. Falling back to accumulating the byte stream",
+                    f"JSON parse exception in decoding output from pip {action}"
+                    "{package}. Falling back to accumulating the byte stream",
                 )
             accumulator = {}
+            parsed = BytesHeaderParser().parsebytes(result.stdout.encode())
             for key, value in parsed.items():
                 accumulator[key] = value
             return accumulator
-        return str(output.decode())
+
+        return result.stdout
+
     except subprocess.CalledProcessError as exc:
         logger.error(
             (
                 f"Failed to {action} {package}\n"
                 f"Exit code: {exc.returncode}\n"
-                f"stdout: {exc.output}"
+                f"stderr: {(exc.stderr or "").strip()}\n"
+                f"stdout: {(exc.stdout or "").strip()}"
             )
         )
-        sys.exit(1)
+        # Re-raise the error or handle it accordingly
+        raise
     except Exception as e:
         logger.error(
-            f"An unexpected exception occurred while try to {action} {package}!",
+            f"An unexpected exception occurred while trying to {action} {package}!",
             e,
         )
-        sys.exit(1)
+        raise
 
 
 def get_site_packages_location() -> str:

guardrails/hub/install.py

@@ -1,7 +1,10 @@
 from contextlib import contextmanager
+import contextlib
 from string import Template
 from typing import Callable, cast, List
 
+import pkg_resources
+
 from guardrails.hub.validator_package_service import (
     ValidatorPackageService,
     ValidatorModuleType,
@@ -52,17 +55,21 @@ def install(
 
     Examples:
         >>> RegexMatch = install("hub://guardrails/regex_match").RegexMatch
+        >>> RegexMatch = install("hub://guardrails/regex_match~=1.4").RegexMatch
+
 
-        >>> install("hub://guardrails/regex_match")
-        >>> import guardrails.hub.regex_match as regex_match
+        >>> install("hub://guardrails/regex_match>=1.4,==1.*.")
+        >>> from guardrails.hub.regex_match import RegexMatch
     """
 
     verbose_printer = console.print
     quiet_printer = console.print if not quiet else lambda x: None
 
     # 1. Validation
     rc_file_exists = RC.exists()
-    module_name = ValidatorPackageService.get_module_name(package_uri)
+    validator_id, validator_version = ValidatorPackageService.get_validator_id(
+        package_uri
+    )
 
     installing_msg = f"Installing {package_uri}..."
     cli_logger.log(
@@ -78,15 +85,15 @@ def install(
     fetch_manifest_msg = "Fetching manifest"
     with loader(fetch_manifest_msg, spinner="bouncingBar"):
         (module_manifest, site_packages) = (
-            ValidatorPackageService.get_manifest_and_site_packages(module_name)
+            ValidatorPackageService.get_manifest_and_site_packages(validator_id)
         )
 
     # 3. Install - Pip Installation of git module
     dl_deps_msg = "Downloading dependencies"
     with loader(dl_deps_msg, spinner="bouncingBar"):
         ValidatorPackageService.install_hub_module(
-            module_manifest,
-            site_packages,
+            validator_id,
+            validator_version=validator_version,
             quiet=quiet,
             upgrade=upgrade,
             logger=cli_logger,
@@ -139,7 +146,16 @@ def install(
     # Print success messages
     cli_logger.info("Installation complete")
 
-    verbose_printer(f"✅Successfully installed {module_name}!\n\n")
+    installed_version_message = ""
+    with contextlib.suppress(Exception):
+        package_name = ValidatorPackageService.get_normalized_package_name(validator_id)
+        installed_version = pkg_resources.get_distribution(package_name).version
+        if installed_version:
+            installed_version_message = f" version {installed_version}"
+
+    verbose_printer(
+        f"✅Successfully installed {validator_id}{installed_version_message}!\n\n"
+    )
     success_message_cli = Template(
         "[bold]Import validator:[/bold]\n"
         "from guardrails.hub import ${export}\n\n"

guardrails/hub/validator_package_service.py

@@ -1,20 +1,22 @@
 import importlib
 import os
 from pathlib import Path
+import re
 import subprocess
 import sys
 
-from typing import List, Literal
+from typing import List, Literal, Optional
 from types import ModuleType
 from pydash.strings import snake_case
+from packaging.utils import canonicalize_name  # PEP 503
 
-from guardrails.classes.generic.stack import Stack
 from guardrails.logger import logger as guardrails_logger
 
 
 from guardrails.cli.hub.utils import pip_process
 from guardrails_hub_types import Manifest
 from guardrails.cli.server.hub_client import get_validator_manifest
+from guardrails.settings import settings
 
 
 json_format: Literal["json"] = "json"
@@ -91,11 +93,13 @@ def get_validator_from_manifest(manifest: Manifest) -> ModuleType:
         Returns:
             Any: The Validator class from the installed module
         """
-        org_package = ValidatorPackageService.get_org_and_package_dirs(manifest)
-        module_name = manifest.module_name
 
-        _relative_path = ".".join([*org_package, module_name])
-        import_line = f"guardrails.hub.{_relative_path}"
+        validator_id = manifest.id
+        import_path = ValidatorPackageService.get_import_path_from_validator_id(
+            validator_id
+        )
+
+        import_line = f"{import_path}"
 
         # Reload or import the module
         return ValidatorPackageService.reload_module(import_line)
@@ -112,14 +116,15 @@ def get_org_and_package_dirs(
 
     @staticmethod
     def add_to_hub_inits(manifest: Manifest, site_packages: str):
+        validator_id = manifest.id
         org_package = ValidatorPackageService.get_org_and_package_dirs(manifest)
         exports: List[str] = manifest.exports or []
         sorted_exports = sorted(exports, reverse=True)
-        module_name = manifest.module_name
-        relative_path = ".".join([*org_package, module_name])
-        import_line = (
-            f"from guardrails.hub.{relative_path} import {', '.join(sorted_exports)}"
+
+        import_path = ValidatorPackageService.get_import_path_from_validator_id(
+            validator_id
         )
+        import_line = f"from {import_path} import {', '.join(sorted_exports)}"
 
         hub_init_location = os.path.join(
             site_packages, "guardrails", "hub", "__init__.py"
@@ -179,14 +184,29 @@ def get_module_path(package_name):
         return package_path
 
     @staticmethod
-    def get_module_name(package_uri: str):
-        if not package_uri.startswith("hub://"):
+    def get_validator_id(validator_uri: str):
+        if not validator_uri.startswith("hub://"):
             raise InvalidHubInstallURL(
                 "Invalid URI! The package URI must start with 'hub://'"
             )
 
-        module_name = package_uri.replace("hub://", "")
-        return module_name
+        validator_uri_with_version = validator_uri.replace("hub://", "")
+
+        validator_id_version_regex = (
+            r"(?P<validator_id>[\/a-zA-Z0-9\-_]+)(?P<version>.*)"
+        )
+        match = re.match(validator_id_version_regex, validator_uri_with_version)
+        validator_version = None
+
+        if match:
+            validator_id = match.group("validator_id")
+            validator_version = (
+                match.group("version").strip() if match.group("version") else None
+            )
+        else:
+            validator_id = validator_uri_with_version
+
+        return (validator_id, validator_version)
 
     @staticmethod
     def get_install_url(manifest: Manifest) -> str:
@@ -207,18 +227,19 @@ def get_install_url(manifest: Manifest) -> str:
     def run_post_install(
         manifest: Manifest, site_packages: str, logger=guardrails_logger
     ):
-        org_package = ValidatorPackageService.get_org_and_package_dirs(manifest)
+        validator_id = manifest.id
         post_install_script = manifest.post_install
+
         if not post_install_script:
             return
 
-        module_name = manifest.module_name
+        import_path = ValidatorPackageService.get_import_path_from_validator_id(
+            validator_id
+        )
+
         relative_path = os.path.join(
             site_packages,
-            "guardrails",
-            "hub",
-            *org_package,
-            module_name,
+            import_path,
             post_install_script,
         )
 
@@ -255,67 +276,52 @@ def get_hub_directory(manifest: Manifest, site_packages: str) -> str:
         org_package = ValidatorPackageService.get_org_and_package_dirs(manifest)
         return os.path.join(site_packages, "guardrails", "hub", *org_package)
 
+    @staticmethod
+    def get_normalized_package_name(validator_id: str):
+        validator_id_parts = validator_id.split("/")
+        concatanated_package_name = (
+            f"{validator_id_parts[0]}-grhub-{validator_id_parts[1]}"
+        )
+        pep_503_package_name = canonicalize_name(concatanated_package_name)
+        return pep_503_package_name
+
+    @staticmethod
+    def get_import_path_from_validator_id(validator_id):
+        pep_503_package_name = ValidatorPackageService.get_normalized_package_name(
+            validator_id
+        )
+        return pep_503_package_name.replace("-", "_")
+
     @staticmethod
     def install_hub_module(
-        module_manifest: Manifest,
-        site_packages: str,
+        validator_id: str,
+        validator_version: Optional[str] = "",
         quiet: bool = False,
         upgrade: bool = False,
         logger=guardrails_logger,
     ):
-        install_url = ValidatorPackageService.get_install_url(module_manifest)
-        install_directory = ValidatorPackageService.get_hub_directory(
-            module_manifest, site_packages
+        pep_503_package_name = ValidatorPackageService.get_normalized_package_name(
+            validator_id
         )
+        validator_version = validator_version if validator_version else ""
+        full_package_name = f"{pep_503_package_name}{validator_version}"
+
+        guardrails_token = settings.rc.token
 
-        pip_flags = [f"--target={install_directory}", "--no-deps"]
+        pip_flags = [
+            f"--index-url=https://__token__:{guardrails_token}@e4c4zula06.execute-api.us-east-1.amazonaws.com/simple",
+            "--extra-index-url=https://pypi.org/simple",
+        ]
 
         if upgrade:
             pip_flags.append("--upgrade")
 
         if quiet:
             pip_flags.append("-q")
 
-        # Install validator module in namespaced directory under guardrails.hub
-        download_output = pip_process("install", install_url, pip_flags, quiet=quiet)
+        # Install from guardrails hub pypi server with public pypi index as fallback
+        download_output = pip_process(
+            "install", full_package_name, pip_flags, quiet=quiet
+        )
         if not quiet:
             logger.info(download_output)
-
-        # Install validator module's dependencies in normal site-packages directory
-        inspect_output = pip_process(
-            "inspect",
-            flags=[f"--path={install_directory}"],
-            format=json_format,
-            quiet=quiet,
-            no_color=True,
-        )
-
-        # throw if inspect_output is a string. Mostly for pyright
-        if isinstance(inspect_output, str):
-            logger.error("Failed to inspect the installed package!")
-            raise FailedPackageInspection
-
-        dependencies = (
-            Stack(*inspect_output.get("installed", []))
-            .at(0, {})
-            .get("metadata", {})  # type: ignore
-            .get("requires_dist", [])  # type: ignore
-        )
-        requirements = list(filter(lambda dep: "extra" not in dep, dependencies))
-        for req in requirements:
-            if "git+" in req:
-                install_spec = req.replace(" ", "")
-                dep_install_output = pip_process("install", install_spec, quiet=quiet)
-                if not quiet:
-                    logger.info(dep_install_output)
-            else:
-                req_info = Stack(*req.split(" "))
-                name = req_info.at(0, "").strip()  # type: ignore
-                versions = req_info.at(1, "").strip("()")  # type: ignore
-                if name:
-                    install_spec = name if not versions else f"{name}{versions}"
-                    dep_install_output = pip_process(
-                        "install", install_spec, quiet=quiet
-                    )
-                    if not quiet:
-                        logger.info(dep_install_output)