Merge remote-tracking branch 'great-expectations/develop' into develop

Author: fredwang1012

.github/workflows/ci.yml

@@ -244,9 +244,6 @@ jobs:
           # Authorize access to Google Cloud with a service account
           ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS
 
-      - name: Install mssql odbc driver
-        run: ./scripts/install_mssql_odbc_driver.sh
-
       - name: Install cloud dependencies
         if: ${{ matrix.start_services }}
         run: |
@@ -710,6 +707,7 @@ jobs:
           invoke deps --gx-install -m '${{ matrix.markers }}' -r test
 
       - name: Install mssql odbc driver
+        if: matrix.markers == 'mssql'
         run: ./scripts/install_mssql_odbc_driver.sh
 
       - name: Configure ECR AWS Credentials

docs/docusaurus/docs/cloud/connect/connect_snowflake.md

@@ -9,7 +9,7 @@ description: Connect GX Cloud to a Snowflake Data Source.
 - You have a [GX Cloud account](https://greatexpectations.io/cloud) with [Workspace Editor permissions](/cloud/access/manage_access.md#roles-and-permissions) or greater.
 - You have a Snowflake database, schema, and table or view.
 - You have a [Snowflake account](https://docs.snowflake.com/en/user-guide-admin) with USAGE privileges on the table or view, database, and schema you are validating, and you have SELECT privileges on the table or view you are validating.
-- You have [configured and stored private and public keys](https://docs.snowflake.com/en/user-guide/key-pair-auth) for Snowflake key-pair authentication.
+- You have [configured and stored an unencrypted private key and public key](https://docs.snowflake.com/en/user-guide/key-pair-auth) for Snowflake key-pair authentication.
    :::warning Password authentication is deprecated
    Snowflake has deprecated password authentication and will remove support for it entirely in the future. Set up new Data Sources with key-pair authentication. If you have older Snowflake Data Sources using password authentication, update them to use key-pair authentication. For more information about the deprecation, see [Snowflake's documentation](https://docs.snowflake.com/en/user-guide/security-mfa-rollout).
    :::
@@ -30,7 +30,7 @@ description: Connect GX Cloud to a Snowflake Data Source.
 
    - **Username**: Enter the username you use to access Snowflake.
 
-   - **Private key**: Enter your RSA private key value. Do not include the start and end markers `-----BEGIN/END ENCRYPTED PRIVATE KEY-----`.
+   - **Private key**: Enter your unencrypted private key value. Do not include the start and end markers `-----BEGIN/END PRIVATE KEY-----`.
 
    - **Database**: Enter the name of the Snowflake database where the data you want to validate is stored. In Snowsight, click **Data** > **Databases**. In the Snowflake Classic Console, click **Databases**.
 

docs/docusaurus/docs/components/_data.jsx

@@ -1,5 +1,5 @@
 export default {
-  release_version: 'great_expectations, version 1.8.0',
+  release_version: 'great_expectations, version 1.8.1',
   min_python: '3.9',
   max_python: '3.13'
 }

docs/docusaurus/docs/core/connect_to_data/configure_credentials/_connection_string.md

@@ -7,7 +7,7 @@ Different types of SQL databases have different formats for their connection det
 
    Other connection string formats are valid provided they are for a SQL database that is supported by SQLAlchemy.  You can find more information on the dialects supported by `SQLAlchemy` on their [dialects](https://docs.sqlalchemy.org/en/20/dialects/index.html) page.
 
-   To connect to Snowflake, you will pass your connection details and credentials as the following parameters: `account`,  `user`, `database`, `schema`, `warehouse`, `role`, and `private_key`. When setting your `private_key` value, do not include the start and end markers `-----BEGIN/END ENCRYPTED PRIVATE KEY-----`.
+   To connect to Snowflake, you will pass your connection details and credentials as the following parameters: `account`,  `user`, `database`, `schema`, `warehouse`, `role`, and `private_key`. When setting your `private_key` value, make sure to use an unencrypted key and do not include the start and end markers `-----BEGIN/END PRIVATE KEY-----`.
 
       :::warning Snowflake password authentication is deprecated
       Snowflake has deprecated password authentication and will remove support for it entirely in the future. Set up new Data Sources with key-pair authentication. If you have older Snowflake Data Sources using password authentication, update them to use key-pair authentication. For more information about the deprecation, see [Snowflake's documentation](https://docs.snowflake.com/en/user-guide/security-mfa-rollout).

docs/docusaurus/docs/core/connect_to_data/sql_data/_create_a_data_source/_create_a_data_source.md

@@ -67,7 +67,7 @@ import DatasourceMethodReferenceTable from './_datasource_method_reference_table
    schema = "my_schema"
    warehouse = "my_wh"
    role = "my_role"
-   private_key = "my_private_key"
+   private_key = "my_unencrypted_private_key"
 
    data_source = context.data_sources.add_snowflake(
       name=datasource_name,

docs/docusaurus/docs/oss/changelog.md

@@ -15,6 +15,24 @@ When we deprecate public functionality, we will
 
 Before we completely remove the functionality in a new major release, there will be at least one minor release that contains the deprecation so that you can smoothly transition.
 
+### 1.8.1
+* [BUGFIX] Fix render_content generation with new row_condition types ([#11481](https://github.com/great-expectations/great_expectations/pull/11481))
+* [BUGFIX] Prevent unmocked HTTP requests in cloud tests ([#11492](https://github.com/great-expectations/great_expectations/pull/11492))
+* [BUGFIX] Update pytest filter to handle google.api_core Python 3.10 end-of-life warning ([#11493](https://github.com/great-expectations/great_expectations/pull/11493))
+* [DOCS] snowflake password deprecation ([#11416](https://github.com/great-expectations/great_expectations/pull/11416))
+* [DOCS] correct Snowflake private key guidance ([#11490](https://github.com/great-expectations/great_expectations/pull/11490))
+* [MAINTENANCE] Stop running python 3.12 marker tests on all PRs ([#11457](https://github.com/great-expectations/great_expectations/pull/11457))
+* [MAINTENANCE] Disable link checker ([#11477](https://github.com/great-expectations/great_expectations/pull/11477))
+* [MAINTENANCE] Transform legacy row_condition string into new Condition object ([#11474](https://github.com/great-expectations/great_expectations/pull/11474))
+* [MAINTENANCE] Keep `condition_parser` field intact for backwards compatibility ([#11484](https://github.com/great-expectations/great_expectations/pull/11484))
+* [MAINTENANCE] Replace broken bitnami image with apache image for spark ([#11444](https://github.com/great-expectations/great_expectations/pull/11444))
+* [MAINTENANCE] Ignore future warning Python 3.9 EOL ([#11486](https://github.com/great-expectations/great_expectations/pull/11486))
+* [MAINTENANCE] Add passthrough path for pandas and spark Condition parser ([#11480](https://github.com/great-expectations/great_expectations/pull/11480))
+* [MAINTENANCE] Constrain `row_condition` groups ([#11488](https://github.com/great-expectations/great_expectations/pull/11488))
+* [MAINTENANCE] Only install mssql drivers when needed ([#11489](https://github.com/great-expectations/great_expectations/pull/11489))
+* [MAINTENANCE] Use Suites v2 REST endpoints ([#11487](https://github.com/great-expectations/great_expectations/pull/11487))
+* [MAINTENANCE] Handle `None` parameter in `Condition`s ([#11491](https://github.com/great-expectations/great_expectations/pull/11491))
+
 ### 1.8.0
 * [FEATURE] Snowflake Key Pair Auth API ([#11395](https://github.com/great-expectations/great_expectations/pull/11395))
 * [BUGFIX] Row condition for Volume Expectations ([#11467](https://github.com/great-expectations/great_expectations/pull/11467))

docs/docusaurus/docusaurus.config.js

@@ -306,7 +306,7 @@ module.exports = {
           lastVersion: 'current',
           versions: {
             current: {
-              label: '1.8.0',
+              label: '1.8.1',
             },
             ['0.18']: {
               label: '0.18.21',

great_expectations/data_context/store/gx_cloud_store_backend.py

@@ -109,7 +109,7 @@ class GXCloudStoreBackend(StoreBackend, metaclass=ABCMeta):
         GXCloudRESTResource.DATA_ASSET: EndpointVersion.V1,
         GXCloudRESTResource.DATA_CONTEXT: EndpointVersion.V1,
         GXCloudRESTResource.DATA_CONTEXT_VARIABLES: EndpointVersion.V1,
-        GXCloudRESTResource.EXPECTATION_SUITE: EndpointVersion.V1,
+        GXCloudRESTResource.EXPECTATION_SUITE: EndpointVersion.V2,
         GXCloudRESTResource.VALIDATION_DEFINITION: EndpointVersion.V1,
         GXCloudRESTResource.VALIDATION_RESULT: EndpointVersion.V1,
     }

great_expectations/deployment_version

@@ -1 +1 @@
-1.8.0
+1.8.1

great_expectations/expectations/conditions.py

@@ -3,7 +3,7 @@
 from enum import Enum
 from typing import TYPE_CHECKING, Any, Iterable, List, Literal, Union
 
-from great_expectations.compatibility.pydantic import BaseModel, Field, validator
+from great_expectations.compatibility.pydantic import BaseModel, Field, root_validator, validator
 from great_expectations.compatibility.typing_extensions import override
 
 if TYPE_CHECKING:
@@ -24,6 +24,39 @@ def __init__(self, value: Any):
         super().__init__(f"Invalid condition type: {type(value)}")
 
 
+class AndContainsOrError(ValueError):
+    """Raised when an AND group contains OR conditions."""
+
+    def __init__(self):
+        super().__init__("AND groups cannot contain OR conditions")
+
+
+class NestedOrError(ValueError):
+    """Raised when OR groups contain nested OR conditions."""
+
+    def __init__(self):
+        super().__init__("OR groups cannot contain nested OR conditions")
+
+
+class TooManyConditionsError(ValueError):
+    """Raised when the number of conditions exceeds the maximum allowed."""
+
+    def __init__(self, count: int, max_conditions: int):
+        super().__init__(
+            f"{max_conditions} conditions is the maximum, but {count} conditions are defined"
+        )
+
+
+class InvalidParameterTypeError(TypeError):
+    """Raised when the type of the parameter is invalid."""
+
+    def __init__(self, parameter: Any, suggestion: str | None = None):
+        message = f"Invalid Parameter type: {type(parameter)}"
+        if suggestion:
+            message += f". {suggestion}"
+        super().__init__(message)
+
+
 class Operator(str, Enum):
     EQUAL = "=="
     NOT_EQUAL = "!="
@@ -45,6 +78,9 @@ def __str__(self) -> str:
 class Column(BaseModel):
     name: str
 
+    def __init__(self, name: str):
+        super().__init__(name=name)
+
     @override
     def __hash__(self) -> int:
         return hash(self.name)
@@ -107,22 +143,12 @@ def dict(self, **kwargs) -> dict:
         return result
 
     def __and__(self, other: Condition) -> AndCondition:
-        new_conditions = []
-        for cond in [self, other]:
-            if isinstance(cond, AndCondition):
-                new_conditions.extend(cond.conditions)
-            else:
-                new_conditions.append(cond)
-        return AndCondition(conditions=new_conditions)
+        """Construct an AndCondition"""
+        return AndCondition(conditions=[self, other])
 
     def __or__(self, other: Condition) -> OrCondition:
-        new_conditions = []
-        for cond in [self, other]:
-            if isinstance(cond, OrCondition):
-                new_conditions.extend(cond.conditions)
-            else:
-                new_conditions.append(cond)
-        return OrCondition(conditions=new_conditions)
+        """Construct an OrCondition"""
+        return OrCondition(conditions=[self, other])
 
 
 class NullityCondition(Condition):
@@ -140,7 +166,21 @@ class ComparisonCondition(Condition):
     type: Literal["comparison"] = Field(default="comparison")
     column: Column
     operator: Operator
-    parameter: Parameter
+    parameter: Parameter = Field(...)
+
+    @root_validator
+    def _validate_parameter_not_none(cls, values):
+        parameter = values.get("parameter")
+        operator = values.get("operator")
+        if parameter is None:
+            if operator == Operator.EQUAL:
+                suggestion = "Did you mean to use Column.is_null()?"
+            elif operator == Operator.NOT_EQUAL:
+                suggestion = "Did you mean to use Column.is_not_null()?"
+            else:
+                suggestion = None
+            raise InvalidParameterTypeError(parameter, suggestion)
+        return values
 
     @override
     def __repr__(self):
@@ -235,3 +275,123 @@ def __repr__(self) -> str:
     PassThroughCondition,
     None,
 ]
+
+
+# Maximum number of conditions allowed in a row_condition
+MAX_CONDITIONS = 100
+
+
+def _count_total_conditions(conditions: List[Condition]) -> int:
+    """Recursively count all conditions including nested ones."""
+    count = 0
+    for condition in conditions:
+        if isinstance(condition, (AndCondition, OrCondition)):
+            count += _count_total_conditions(condition.conditions)
+        else:
+            count += 1
+    return count
+
+
+def _flatten_and_conditions(conditions: List[Condition]) -> List[Condition]:
+    """Flatten nested AndConditions within a list of conditions."""
+    flattened = []
+    for condition in conditions:
+        if isinstance(condition, AndCondition):
+            # Recursively flatten nested AndConditions
+            flattened.extend(_flatten_and_conditions(condition.conditions))
+        else:
+            flattened.append(condition)
+    return flattened
+
+
+def _contains_or_in_and(condition: Condition) -> bool:
+    """Check if an AndCondition contains any OrConditions."""
+    if isinstance(condition, AndCondition):
+        for cond in condition.conditions:
+            if isinstance(cond, OrCondition):
+                return True
+            # Recursively check nested AndConditions
+            if isinstance(cond, AndCondition) and _contains_or_in_and(cond):
+                return True
+    return False
+
+
+def _contains_nested_or(condition: Condition) -> bool:
+    """Check if an OrCondition contains nested OrConditions."""
+    if isinstance(condition, OrCondition):
+        for cond in condition.conditions:
+            if isinstance(cond, OrCondition):
+                return True
+            # Also check if any nested AndConditions contain OrConditions
+            # (which would create a nested OR structure)
+            if isinstance(cond, AndCondition):
+                for and_cond in cond.conditions:
+                    if isinstance(and_cond, OrCondition):
+                        # This is an OR within an AND within an OR, which creates nested ORs
+                        return True
+    return False
+
+
+def _validate_and_condition(row_condition: AndCondition) -> AndCondition:
+    """Validate AndCondition constraints."""
+    # Check for OrConditions within AndConditions
+    if _contains_or_in_and(row_condition):
+        raise AndContainsOrError()
+
+    # Flatten nested AndConditions
+    # We only flatten nested AndConditions because
+    # logical ANDs are associative, while logical ORs are non-associative.
+    flattened_conditions = _flatten_and_conditions(row_condition.conditions)
+
+    # Return flattened AndCondition
+    return AndCondition(conditions=flattened_conditions)
+
+
+def _validate_or_condition(row_condition: OrCondition) -> OrCondition:
+    """Validate OrCondition constraints."""
+    # Check for nested OrConditions
+    if _contains_nested_or(row_condition):
+        raise NestedOrError()
+
+    return row_condition
+
+
+def validate_row_condition(row_condition: RowConditionType) -> RowConditionType:
+    """Validate row_condition according to GX Cloud UI constraints.
+
+    1. Flatten nested AndConditions within AndConditions
+    2. Raise error if OrConditions are nested within AndConditions
+    3. Raise error if OrConditions are nested within OrConditions
+    4. Raise error if total conditions exceed MAX_CONDITIONS
+
+    Args:
+        row_condition: The row condition to validate
+
+    Returns:
+        The validated (and possibly flattened) row condition
+
+    Raises:
+        ValueError: If the row condition violates any constraints
+    """
+    # String conditions and None are always valid
+    if row_condition is None or isinstance(row_condition, str):
+        return row_condition
+
+    # Single condition types are always valid
+    if isinstance(row_condition, (ComparisonCondition, NullityCondition, PassThroughCondition)):
+        return row_condition
+
+    # Validate total condition count
+    total_count = _count_total_conditions(row_condition.conditions)
+    if total_count > MAX_CONDITIONS:
+        raise TooManyConditionsError(total_count, MAX_CONDITIONS)
+
+    # Validate AndCondition
+    if isinstance(row_condition, AndCondition):
+        return _validate_and_condition(row_condition)
+
+    # Validate OrCondition
+    if isinstance(row_condition, OrCondition):
+        return _validate_or_condition(row_condition)
+
+    raise InvalidConditionTypeError(row_condition)