tests/fix: more comprehensive api object detection

Author: tazlin

horde_sdk/ai_horde_api/endpoints.py

@@ -97,17 +97,17 @@ class AI_HORDE_API_ENDPOINT_SUBPATH(GENERIC_API_ENDPOINT_SUBPATH):
     v2_documents_sponsors = "/v2/documents/sponsors"
     v2_documents_terms = "/v2/documents/terms"
 
-    # v2_styles_image_by_name = "/v2/styles/image_by_name/{style_name}"
-    # v2_styles_image_by_id = "/v2/styles/image/{style_id}"
-    # v2_collections_by_name = "/v2/collection_by_name/{collection_name}"
-    # v2_styles_image_example_by_id = "/v2/styles/image/{style_id}/example"
-    # v2_styles_text_by_id = "/v2/styles/text/{style_id}"
-    # v2_styles_image = "/v2/styles/image"
-    # v2_collections_by_id = "/v2/collections/{collection_id}"
-    # v2_styles_image_example_by_id_example = "/v2/styles/image/{style_id}/example/{example_id}"
-    # v2_styles_text = "/v2/styles/text"
-    # v2_styles_text_by_name = "/v2/styles/text_by_name/{style_name}"
-    # v2_collections = "/v2/collections"
+    v2_styles_image_by_name = "/v2/styles/image_by_name/{style_name}"
+    v2_styles_image_by_id = "/v2/styles/image/{style_id}"
+    v2_collections_by_name = "/v2/collection_by_name/{collection_name}"
+    v2_styles_image_example_by_id = "/v2/styles/image/{style_id}/example"
+    v2_styles_text_by_id = "/v2/styles/text/{style_id}"
+    v2_styles_image = "/v2/styles/image"
+    v2_collections_by_id = "/v2/collections/{collection_id}"
+    v2_styles_image_example_by_id_example = "/v2/styles/image/{style_id}/example/{example_id}"
+    v2_styles_text = "/v2/styles/text"
+    v2_styles_text_by_name = "/v2/styles/text_by_name/{style_name}"
+    v2_collections = "/v2/collections"
 
 
 def get_ai_horde_swagger_url() -> str:
@@ -116,3 +116,25 @@ def get_ai_horde_swagger_url() -> str:
         base_url=AI_HORDE_BASE_URL,
         path=AI_HORDE_API_ENDPOINT_SUBPATH.swagger,
     )
+
+
+def get_admin_only_endpoints() -> set[str]:
+    """Return all of the endpoints that are admin-only."""
+    return {
+        AI_HORDE_API_ENDPOINT_SUBPATH.v2_status_modes,
+        AI_HORDE_API_ENDPOINT_SUBPATH.v2_filters,
+        AI_HORDE_API_ENDPOINT_SUBPATH.v2_filters_regex,
+        AI_HORDE_API_ENDPOINT_SUBPATH.v2_filters_regex_single,
+        AI_HORDE_API_ENDPOINT_SUBPATH.v2_operations_block_worker_ipaddr_single,
+        AI_HORDE_API_ENDPOINT_SUBPATH.v2_operations_ipaddr,
+        AI_HORDE_API_ENDPOINT_SUBPATH.v2_operations_ipaddr_single,
+        AI_HORDE_API_ENDPOINT_SUBPATH.v2_kudos_award,
+    }
+
+
+def get_deprecated_endpoints() -> set[str]:
+    """Return all of the endpoints that are deprecated."""
+    return {
+        AI_HORDE_API_ENDPOINT_SUBPATH.v2_generate_pop_multi,
+        AI_HORDE_API_ENDPOINT_SUBPATH.v2_generate_rate_id,
+    }

horde_sdk/meta.py

@@ -74,7 +74,35 @@ def any_unimported_classes(module: types.ModuleType, super_type: type) -> tuple[
     return bool(missing_classes), missing_classes
 
 
-def all_undefined_classes(module: types.ModuleType) -> dict[str, str]:
+def all_undefined_classes(module: types.ModuleType) -> list[str]:
+    """Return all of the models defined on the API but not in the SDK."""
+    module_found_classes = find_subclasses(module, HordeAPIObject)
+
+    defined_api_object_names: set[str] = set()
+
+    for class_type in module_found_classes:
+        if not issubclass(class_type, HordeAPIObject):
+            raise TypeError(f"Expected {class_type} to be a HordeAPIObject")
+
+        api_model_name = class_type.get_api_model_name()
+        if api_model_name is not None:
+            defined_api_object_names.add(api_model_name)
+
+    undefined_classes: list[str] = []
+
+    parser = SwaggerParser(swagger_doc_url=get_ai_horde_swagger_url())
+    swagger_doc = parser.get_swagger_doc()
+
+    all_api_objects = set(swagger_doc.definitions.keys())
+    missing_object_names = all_api_objects - defined_api_object_names
+
+    for object_name in missing_object_names:
+        undefined_classes.append(object_name)
+
+    return undefined_classes
+
+
+def all_undefined_classes_for_endpoints(module: types.ModuleType) -> dict[str, str]:
     """Return all of the models defined on the API but not in the SDK."""
     module_found_classes = find_subclasses(module, HordeAPIObject)
 
@@ -132,7 +160,6 @@ def all_unaddressed_endpoints_ai_horde() -> set[AI_HORDE_API_ENDPOINT_SUBPATH]:
 
     for path in known_paths:
         if path not in all_classes_paths:
-            print(f"Unaddressed path: {path}")
             unaddressed_paths.add(path)
 
     return unaddressed_paths

tests/test_verify_api_surface.py

@@ -43,10 +43,48 @@ def test_all_ai_horde_api_data_objects_imported() -> None:
 @pytest.mark.object_verify
 def test_all_ai_horde_api_models_defined() -> None:
     import horde_sdk.ai_horde_api.apimodels
-    from horde_sdk.meta import all_undefined_classes
+    from horde_sdk.meta import all_undefined_classes, all_undefined_classes_for_endpoints
 
     undefined_classes = all_undefined_classes(horde_sdk.ai_horde_api.apimodels)
 
+    # all_undefined_classes_for_endpoints handles the ones directly referenced by endpoints, so we remove them
+    undefined_classes_for_endpoints = all_undefined_classes_for_endpoints(horde_sdk.ai_horde_api.apimodels)
+    for key in undefined_classes_for_endpoints:
+        if key in undefined_classes:
+            undefined_classes.remove(key)
+
+    assert (
+        "GenerationInputStable" not in undefined_classes
+    ), "A model which is known to be defined in the SDK was not found. Something critically bad has happened."
+
+    # Pretty print the undefined classes sorted by dict values, NOT by keys
+    import json
+
+    error_responses = {
+        "RequestError",
+        "RequestValidationError",
+    }
+
+    for error_response in error_responses:
+        if error_response in undefined_classes:
+            print(f"Warning: {error_response} is an error response which may not be handled.")
+            undefined_classes.remove(error_response)
+
+    undefined_classes_sorted = sorted(undefined_classes)
+    print(json.dumps(undefined_classes_sorted, indent=4))
+
+    assert not undefined_classes, (
+        "The following models are defined in the API but not in the SDK: " f"{undefined_classes}"
+    )
+
+
+@pytest.mark.object_verify
+def test_all_ai_horde_api_models_defined_for_endpoints() -> None:
+    import horde_sdk.ai_horde_api.apimodels
+    from horde_sdk.meta import all_undefined_classes_for_endpoints
+
+    undefined_classes = all_undefined_classes_for_endpoints(horde_sdk.ai_horde_api.apimodels)
+
     assert (
         "GenerationInputStable" not in undefined_classes
     ), "A model which is known to be defined in the SDK was not found. Something critically bad has happened."
@@ -86,10 +124,19 @@ def test_all_ai_horde_endpoints_known() -> None:
 # @pytest.mark.skip(reason="This test is not yet enforced.")
 @pytest.mark.object_verify
 def test_all_ai_horde_endpoints_addressed() -> None:
+    from horde_sdk.ai_horde_api.endpoints import get_admin_only_endpoints, get_deprecated_endpoints
     from horde_sdk.meta import all_unaddressed_endpoints_ai_horde
 
     unaddressed_endpoints = all_unaddressed_endpoints_ai_horde()
 
+    all_ignored_endpoints = get_admin_only_endpoints() | get_deprecated_endpoints()
+
+    unaddressed_endpoints -= all_ignored_endpoints
+
+    print()
+    for unaddressed_endpoint in unaddressed_endpoints:
+        print(f"Unaddressed path: {unaddressed_endpoint}.")
+
     assert not unaddressed_endpoints, (
         "The following endpoints are defined in the API but not in the SDK: " f"{unaddressed_endpoints}"
     )