Resolve Overrides (#37503)

* address an issue with not properly honoring the override variables which skip individual tox environments at runtime

Author: scbedd

scripts/devops_tasks/test_regression.py

@@ -16,7 +16,6 @@
 import logging
 from common_tasks import (
     run_check_call,
-    parse_require,
     install_package_from_whl,
     filter_dev_requirements,
     find_packages_missing_on_pypi,
@@ -32,8 +31,9 @@
     clone_repo,
 )
 
-from ci_tools.functions import discover_targeted_packages, str_to_bool, find_whl
-from ci_tools.parsing import ParsedSetup
+from ci_tools.functions import discover_targeted_packages, find_whl
+from ci_tools.parsing import ParsedSetup, parse_require
+from ci_tools.variables import str_to_bool
 
 AZURE_GLOB_STRING = "azure*"
 
@@ -407,7 +407,7 @@ def run_main(args):
         "--service",
         help=("Name of service directory (under sdk/) to test." "Example: --service applicationinsights"),
     )
-    
+
     parser.add_argument(
         "--dependent-service",
         dest="dependent_service",

tools/azure-sdk-tools/ci_tools/build.py

@@ -3,10 +3,9 @@
 from subprocess import run
 
 from typing import List
-from ci_tools.functions import discover_targeted_packages, str_to_bool, process_requires
+from ci_tools.functions import discover_targeted_packages, process_requires
 from ci_tools.parsing import ParsedSetup
-from ci_tools.variables import DEFAULT_BUILD_ID
-from ci_tools.variables import discover_repo_root, get_artifact_directory
+from ci_tools.variables import DEFAULT_BUILD_ID, str_to_bool, discover_repo_root, get_artifact_directory
 from ci_tools.versioning.version_shared import set_version_py, set_dev_classifier
 from ci_tools.versioning.version_set_dev import get_dev_version, format_build_id
 from ci_tools.logging import initialize_logger, run_logged

tools/azure-sdk-tools/ci_tools/conda/CondaConfiguration.py

@@ -1,8 +1,8 @@
-from typing import List, Any
+from typing import List, Any, Optional
 import os
 import bs4
 import urllib3
-from ci_tools.functions import str_to_bool
+from ci_tools.variables import str_to_bool
 
 http = urllib3.PoolManager()
 # arguments: |
@@ -124,7 +124,7 @@ def __init__(
         common_root: str,
         in_batch: bool,
         checkout: List[CheckoutConfiguration],
-        created_sdist_path: str = None,
+        created_sdist_path: Optional[str] = None,
         service: str = "",
         conda_py_versions: List[str] = [],
         channels: List[str] = [],
@@ -133,17 +133,17 @@ def __init__(
         self.common_root: str = common_root
         self.in_batch: bool = in_batch
         self.checkout: List[CheckoutConfiguration] = checkout
-        self.created_sdist_path: str = created_sdist_path
+        self.created_sdist_path: Optional[str] = created_sdist_path
         self.service: str = service
         self.conda_py_versions = conda_py_versions
         self.channels = channels
 
     @classmethod
     def from_json(cls, raw_json_blob: dict):
-        name = raw_json_blob.get("name")
+        name = raw_json_blob.get("name", None)
         common_root = raw_json_blob.get("common_root", None)
         in_batch = str_to_bool(raw_json_blob["in_batch"])
-        checkout_config = parse_checkout_config(raw_json_blob.get("checkout"))
+        checkout_config = parse_checkout_config(raw_json_blob.get("checkout", []))
         conda_py_versions = raw_json_blob.get("conda_py_versions", [])
         service = raw_json_blob.get("service", None)
         channels = raw_json_blob.get("channels", [])

tools/azure-sdk-tools/ci_tools/functions.py

@@ -9,7 +9,7 @@
 from packaging.version import Version, parse, InvalidVersion
 from pkg_resources import Requirement
 
-from ci_tools.variables import discover_repo_root, DEV_BUILD_IDENTIFIER
+from ci_tools.variables import discover_repo_root, DEV_BUILD_IDENTIFIER, str_to_bool
 from ci_tools.parsing import ParsedSetup, get_config_setting
 from pypi_tools.pypi import PyPIClient
 
@@ -134,20 +134,6 @@ def compare_python_version(version_spec: str) -> bool:
     return current_sys_version in spec_set
 
 
-def str_to_bool(input_string: str) -> bool:
-    """
-    Takes a boolean string representation and returns a bool type value.
-    """
-    if isinstance(input_string, bool):
-        return input_string
-    elif input_string.lower() in ("true", "t", "1"):
-        return True
-    elif input_string.lower() in ("false", "f", "0"):
-        return False
-    else:
-        return False
-
-
 def generate_difference(original_packages: List[str], filtered_packages: List[str]):
     return list(set(original_packages) - set(filtered_packages))
 

tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py

@@ -26,6 +26,7 @@
 from packaging.specifiers import SpecifierSet
 from setuptools import Extension
 
+from ci_tools.variables import str_to_bool
 
 NEW_REQ_PACKAGES = ["azure-core", "azure-mgmt-core"]
 
@@ -146,9 +147,10 @@ def get_config_setting(package_path: str, setting: str, default: Any = True) ->
     pyproject.toml. If the input 'setting' does NOT exist, the provided default value will be returned.
     """
     # we should always take the override if one is present
-    override_value = os.getenv(f"{os.path.basename(package_path).upper()}_{setting.upper()}", None)
+    override_value = os.getenv(f"{os.path.basename(package_path).upper().replace('-','_')}_{setting.upper()}", None)
+
     if override_value:
-        return override_value
+        return str_to_bool(override_value)
 
     # if no override, check for the config setting in the pyproject.toml
     config = get_build_config(package_path)

tools/azure-sdk-tools/ci_tools/variables.py

@@ -1,5 +1,18 @@
 import os
 
+def str_to_bool(input_string: str) -> bool:
+    """
+    Takes a boolean string representation and returns a bool type value.
+    """
+    if isinstance(input_string, bool):
+        return input_string
+    elif input_string.lower() in ("true", "t", "1"):
+        return True
+    elif input_string.lower() in ("false", "f", "0"):
+        return False
+    else:
+        return False
+
 
 def discover_repo_root(input_repo: str = None):
     """

tools/azure-sdk-tools/tests/test_pyproject_interactions.py

@@ -4,6 +4,7 @@
 import pytest
 
 from ci_tools.parsing import update_build_config, get_build_config, get_config_setting
+from ci_tools.environment_exclusions import is_check_enabled
 
 integration_folder = os.path.join(os.path.dirname(__file__), "integration")
 pyproject_folder = os.path.join(integration_folder, "scenarios", "sample_pyprojects")
@@ -58,6 +59,22 @@ def test_nonpresent_pyproject_update():
         reloaded_build_config = get_build_config(new_path)
         assert reloaded_build_config == update_result
 
+@pytest.mark.parametrize(
+    "check_name, environment_value, expected_result",
+    [
+        ("mindependency", "true", True),
+        ("mindependency", "false", False)
+    ]
+)
+def test_environment_override(check_name, environment_value, expected_result):
+    with tempfile.TemporaryDirectory() as temp_dir:
+        new_path = shutil.copy(pyproject_file, temp_dir)
+
+        os.environ[f"{os.path.basename(temp_dir).upper()}_{check_name.upper()}"] = environment_value
+        result = is_check_enabled(temp_dir, check_name)
+
+        assert result == expected_result
+
 
 def test_pyproject_update_check_override():
     with tempfile.TemporaryDirectory() as temp_dir: