Add DSR Concepts to Fideslang [#94] (#95)

- Add < and > as allowed errors to FidesKey
- Override FidesValidationError to be a subclass of ValueError not Exception so Pydantic can pick up and raise as a ValidationError.
- Add fides_meta fields to Dataset, Collection, and DatasetField. These largely contain concepts for handling DSR's in fides.
- Dataset.fidesctl_meta has been renamed to Dataset.fides_meta.
- Add type validation.
- Allow both fidesops_meta and fides_meta to be specified on Dataset/Collection/DatasetField for backwards-compatibility but have instantiation move fidesops_meta to fides_meta.

Co-authored-by: Andrew Jackson <[email protected]>

Author: pattisdr

CHANGELOG.md

@@ -16,6 +16,11 @@ The types of changes are:
 
 ## [Unreleased](https://github.com/ethyca/fideslang/compare/1.3.1...main)
 
+### Changed
+
+* Moved over DSR concepts into Fideslang. Expanded allowable characters for FideKey and added additional Dataset validation. [#95](https://github.com/ethyca/fideslang/pull/95)
+
+
 ## [1.3.1](https://github.com/ethyca/fideslang/compare/1.3.0...1.3.1)
 
 ### Fixed

Makefile

@@ -88,7 +88,7 @@ mypy:
 pylint:
 	@$(RUN) pylint src/
 
-pytest:
+pytest: build-local
 	@$(RUN) pytest -x
 
 xenon:

src/fideslang/__init__.py

@@ -18,7 +18,10 @@
     DataSubject,
     DataUse,
     Evaluation,
+    FidesMeta,
     FidesModel,
+    FidesDatasetReference,
+    FidesCollectionKey,
     Organization,
     Policy,
     PolicyRule,

src/fideslang/models.py

@@ -1,20 +1,33 @@
+# pylint: disable=too-many-lines
+
 """
 Contains all of the Fides resources modeled as Pydantic models.
 """
 from __future__ import annotations
 
 from enum import Enum
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Literal, Optional, Union
 from warnings import warn
 
-from pydantic import AnyUrl, BaseModel, Field, HttpUrl, root_validator, validator
+from pydantic import (
+    AnyUrl,
+    BaseModel,
+    ConstrainedStr,
+    Field,
+    HttpUrl,
+    PositiveInt,
+    root_validator,
+    validator,
+)
 
 from fideslang.validation import (
     FidesKey,
     check_valid_country_code,
     matching_parent_key,
     no_self_reference,
+    parse_data_type_string,
     sort_list_objects_by_name,
+    valid_data_type,
 )
 
 # Reusable components
@@ -285,23 +298,162 @@ class MyDatasetField(DatasetFieldBase):
     )
 
 
-class DatasetField(DatasetFieldBase):
+EdgeDirection = Literal["from", "to"]
+
+
+class FidesDatasetReference(BaseModel):
+    """Reference to a field from another Collection"""
+
+    dataset: FidesKey
+    field: str
+    direction: Optional[EdgeDirection]
+
+
+class FidesMeta(BaseModel):
+    """Supplementary metadata used by the Fides application for additional features."""
+
+    references: Optional[List[FidesDatasetReference]] = Field(
+        description="Fields that current field references or is referenced by. Used for drawing the edges of a DSR graph.",
+        default=None,
+    )
+    identity: Optional[str] = Field(
+        description="The type of the identity data that should be used to query this collection for a DSR."
+    )
+    primary_key: Optional[bool] = Field(
+        description="Whether the current field can be considered a primary key of the current collection"
+    )
+    data_type: Optional[str] = Field(
+        description="Optionally specify the data type. Fides will attempt to cast values to this type when querying."
+    )
+    length: Optional[PositiveInt] = Field(
+        description="Optionally specify the allowable field length. Fides will not generate values that exceed this size."
+    )
+    return_all_elements: Optional[bool] = Field(
+        description="Optionally specify to query for the entire array if the array is an entrypoint into the node. Default is False."
+    )
+    read_only: Optional[bool] = Field(
+        description="Optionally specify if a field is read-only, meaning it can't be updated or deleted."
+    )
+
+    @validator("data_type")
+    @classmethod
+    def valid_data_type(cls, value: Optional[str]) -> Optional[str]:
+        """Validate that all annotated data types exist in the taxonomy"""
+        return valid_data_type(value)
+
+
+class FidesopsMetaBackwardsCompat(BaseModel):
+    """Mixin to convert fidesops_meta to fides_meta for backwards compatibility
+    as we add DSR concepts to fideslang"""
+
+    def __init__(self, **data: Union[Dataset, DatasetCollection, DatasetField]) -> None:
+        """For Datasets, DatasetCollections, and DatasetFields, if old fidesops_meta field is specified,
+        convert this to a fides_meta field instead."""
+        fidesops_meta = data.pop("fidesops_meta", None)
+        fides_meta = data.pop("fides_meta", None)
+        super().__init__(
+            fides_meta=fides_meta or fidesops_meta,
+            **data,
+        )
+
+
+class DatasetField(DatasetFieldBase, FidesopsMetaBackwardsCompat):
     """
     The DatasetField resource model.
 
     This resource is nested within a DatasetCollection.
     """
 
+    fides_meta: Optional[FidesMeta] = None
+
     fields: Optional[List[DatasetField]] = Field(
         description="An optional array of objects that describe hierarchical/nested fields (typically found in NoSQL databases).",
     )
 
+    @validator("fides_meta")
+    @classmethod
+    def valid_meta(cls, meta_values: Optional[FidesMeta]) -> Optional[FidesMeta]:
+        """Validate upfront that the return_all_elements flag can only be specified on array fields"""
+        if not meta_values:
+            return meta_values
+
+        is_array: bool = bool(
+            meta_values.data_type and meta_values.data_type.endswith("[]")
+        )
+        if not is_array and meta_values.return_all_elements is not None:
+            raise ValueError(
+                "The 'return_all_elements' attribute can only be specified on array fields."
+            )
+        return meta_values
+
+    @validator("fields")
+    @classmethod
+    def validate_object_fields(  # type: ignore
+        cls,
+        fields: Optional[List["DatasetField"]],
+        values: Dict[str, Any],
+    ) -> Optional[List["DatasetField"]]:
+        """Two validation checks for object fields:
+        - If there are sub-fields specified, type should be either empty or 'object'
+        - Additionally object fields cannot have data_categories.
+        """
+        declared_data_type = None
+        field_name: str = values.get("name")  # type: ignore
+
+        if values.get("fides_meta"):
+            declared_data_type = values["fides_meta"].data_type
+
+        if fields and declared_data_type:
+            data_type, _ = parse_data_type_string(declared_data_type)
+            if data_type != "object":
+                raise ValueError(
+                    f"The data type '{data_type}' on field '{field_name}' is not compatible with specified sub-fields. Convert to an 'object' field."
+                )
+
+        if (fields or declared_data_type == "object") and values.get("data_categories"):
+            raise ValueError(
+                f"Object field '{field_name}' cannot have specified data_categories. Specify category on sub-field instead"
+            )
+
+        return fields
+
+
+# this is required for the recursive reference in the pydantic model:
+DatasetField.update_forward_refs()
+
+
+class FidesCollectionKey(ConstrainedStr):
+    """
+    Dataset.Collection name where both dataset and collection names are valid FidesKeys
+    """
+
+    @classmethod
+    def validate(cls, value: str) -> str:
+        """
+        Overrides validation to check FidesCollectionKey format, and that both the dataset
+        and collection names have the FidesKey format.
+        """
+        values = value.split(".")
+        if len(values) == 2:
+            FidesKey.validate(values[0])
+            FidesKey.validate(values[1])
+            return value
+        raise ValueError(
+            "FidesCollection must be specified in the form 'FidesKey.FidesKey'"
+        )
+
 
-class DatasetCollection(BaseModel):
+class CollectionMeta(BaseModel):
+    """Collection-level specific annotations used for query traversal"""
+
+    after: Optional[List[FidesCollectionKey]]
+
+
+class DatasetCollection(FidesopsMetaBackwardsCompat):
     """
     The DatasetCollection resource model.
 
-    This resource is nested witin a Dataset.
+    This resource is nested within a Dataset.
     """
 
     name: str = name_field
@@ -320,6 +472,8 @@ class DatasetCollection(BaseModel):
         description="An array of objects that describe the collection's fields.",
     )
 
+    fides_meta: Optional[CollectionMeta] = None
+
     _sort_fields: classmethod = validator("fields", allow_reuse=True)(
         sort_list_objects_by_name
     )
@@ -362,10 +516,11 @@ class DatasetMetadata(BaseModel):
     """
 
     resource_id: Optional[str]
+    after: Optional[List[FidesKey]]
 
 
-class Dataset(FidesModel):
-    "The Dataset resource model."
+class Dataset(FidesModel, FidesopsMetaBackwardsCompat):
+    """The Dataset resource model."""
 
     meta: Optional[Dict[str, str]] = Field(
         description="An optional object that provides additional information about the Dataset. You can structure the object however you like. It can be a simple set of `key: value` properties or a deeply nested hierarchy of objects. How you use the object is up to you: Fides ignores it."
@@ -377,8 +532,8 @@ class Dataset(FidesModel):
         default="aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified",
         description="Array of Data Qualifier resources identified by `fides_key`, that apply to all collections in the Dataset.",
     )
-    fidesctl_meta: Optional[DatasetMetadata] = Field(
-        description=DatasetMetadata.__doc__,
+    fides_meta: Optional[DatasetMetadata] = Field(
+        description=DatasetMetadata.__doc__, default=None
     )
     joint_controller: Optional[ContactDetails] = Field(
         description=ContactDetails.__doc__,
@@ -393,6 +548,7 @@ class Dataset(FidesModel):
     collections: List[DatasetCollection] = Field(
         description="An array of objects that describe the Dataset's collections.",
     )
+
     _sort_collections: classmethod = validator("collections", allow_reuse=True)(
         sort_list_objects_by_name
     )
@@ -848,7 +1004,8 @@ def deprecate_system_dependencies(cls, value: List[FidesKey]) -> List[FidesKey]:
         return value
 
     class Config:
-        "Class for the System config"
+        """Class for the System config"""
+
         use_enum_values = True
 
 

src/fideslang/validation.py

@@ -3,17 +3,16 @@
 """
 
 import re
-from typing import List, Dict, Pattern
+from typing import Dict, List, Optional, Pattern, Set, Tuple
 
 from pydantic import ConstrainedStr
 
 from fideslang.default_fixtures import COUNTRY_CODES
 
-
 VALID_COUNTRY_CODES = [country["alpha3Code"] for country in COUNTRY_CODES]
 
 
-class FidesValidationError(Exception):
+class FidesValidationError(ValueError):
     """Custom exception for when the pydantic ValidationError can't be used."""
 
 
@@ -22,13 +21,15 @@ class FidesKey(ConstrainedStr):
     A FidesKey type that creates a custom constrained string.
     """
 
-    regex: Pattern[str] = re.compile(r"^[a-zA-Z0-9_.-]+$")
+    regex: Pattern[str] = re.compile(r"^[a-zA-Z0-9_.<>-]+$")
 
     @classmethod  # This overrides the default method to throw the custom FidesValidationError
     def validate(cls, value: str) -> str:
+        """Throws ValueError if val is not a valid FidesKey"""
+
         if not cls.regex.match(value):
             raise FidesValidationError(
-                f"FidesKeys must only contain alphanumeric characters, '.', '_' or '-'. Value provided: {value}"
+                f"FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'. Value provided: {value}"
             )
 
         return value
@@ -73,7 +74,7 @@ def matching_parent_key(value: FidesKey, values: Dict) -> FidesKey:
     parent_key_from_fides_key = ".".join(split_fides_key[:-1])
     if parent_key_from_fides_key != value:
         raise FidesValidationError(
-            "The parent_key ({0}) does not match the parent parsed ({1}) from the fides_key ({2})!".format(
+            "The parent_key ({0}) does match the parent parsed ({1}) from the fides_key ({2})!".format(
                 value, parent_key_from_fides_key, fides_key
             )
         )
@@ -93,3 +94,46 @@ def check_valid_country_code(country_code_list: List) -> List:
                     )
                 )
     return country_code_list
+
+
+def parse_data_type_string(type_string: Optional[str]) -> Tuple[Optional[str], bool]:
+    """Parse the data type string. Arrays are expressed in the form 'type[]'.
+
+    e.g.
+    - 'string' -> ('string', false)
+    - 'string[]' -> ('string', true)
+
+    These data_types are for use in DatasetField.fides_meta.
+    """
+    if not type_string:
+        return None, False
+    idx = type_string.find("[]")
+    if idx == -1:
+        return type_string, False
+    return type_string[:idx], True
+
+
+# Data types that Fides is currently configured to handle
+DATA_TYPE_NAMES: Set[str] = {
+    "string",
+    "integer",
+    "float",
+    "boolean",
+    "object_id",
+    "object",
+}
+
+
+def is_valid_data_type(type_name: str) -> bool:
+    """Is this type a valid data type identifier in fides?"""
+    return type_name is None or type_name in DATA_TYPE_NAMES
+
+
+def valid_data_type(data_type_str: Optional[str]) -> Optional[str]:
+    """If the data_type is provided ensure that it is a member of DataType."""
+
+    parsed_date_type, _ = parse_data_type_string(data_type_str)
+    if not is_valid_data_type(parsed_date_type):  # type: ignore
+        raise ValueError(f"The data type {data_type_str} is not supported.")
+
+    return data_type_str

tests/conftest.py

@@ -53,9 +53,7 @@ def resources_dict():
                             name="Email",
                             description="User's Email",
                             path="another.another.path",
-                            data_categories=[
-                                "user.contact.email"
-                            ],
+                            data_categories=["user.contact.email"],
                             data_qualifier="aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified",
                         ),
                     ],