feat: UTC-416: Implement 'Ensure all annotators are evaluated' setting for GT tasks

reference.md

@@ -8961,6 +8961,14 @@ client.projects.create()
 <dl>
 <dd>
 
+**annotator_evaluation_enabled:** `typing.Optional[bool]` — Enable annotator evaluation for the project
+
+</dd>
+</dl>
+
+<dl>
+<dd>
+
 **color:** `typing.Optional[str]` 
 
 </dd>
@@ -9583,6 +9591,14 @@ client.projects.update(
 <dl>
 <dd>
 
+**annotator_evaluation_enabled:** `typing.Optional[bool]` — Enable annotator evaluation for the project
+
+</dd>
+</dl>
+
+<dl>
+<dd>
+
 **annotator_evaluation_minimum_score:** `typing.Optional[str]` 
 
 </dd>
@@ -9599,6 +9615,14 @@ client.projects.update(
 <dl>
 <dd>
 
+**annotator_evaluation_onboarding_tasks:** `typing.Optional[int]` 
+
+</dd>
+</dl>
+
+<dl>
+<dd>
+
 **assignment_settings:** `typing.Optional[AssignmentSettingsRequest]` 
 
 </dd>
@@ -34481,23 +34505,23 @@ client.projects.metrics.custom.update_lambda(
 <dl>
 <dd>
 
-**code:** `str` 
+**code:** `str` — The Python code for the custom metric function.
 
 </dd>
 </dl>
 
 <dl>
 <dd>
 
-**region:** `typing.Optional[str]` 
+**region:** `typing.Optional[str]` — The AWS region for the Lambda function. Uses default if not provided.
 
 </dd>
 </dl>
 
 <dl>
 <dd>
 
-**role:** `typing.Optional[str]` 
+**role:** `typing.Optional[str]` — The AWS IAM role ARN for the Lambda function. Uses default if not provided.
 
 </dd>
 </dl>

src/label_studio_sdk/projects/client.py

@@ -191,6 +191,7 @@ def list(
     def create(
         self,
         *,
+        annotator_evaluation_enabled: typing.Optional[bool] = OMIT,
         color: typing.Optional[str] = OMIT,
         control_weights: typing.Optional[typing.Optional[typing.Any]] = OMIT,
         created_by: typing.Optional[UserSimpleRequest] = OMIT,
@@ -227,6 +228,9 @@ def create(
 
         Parameters
         ----------
+        annotator_evaluation_enabled : typing.Optional[bool]
+            Enable annotator evaluation for the project
+
         color : typing.Optional[str]
 
         control_weights : typing.Optional[typing.Optional[typing.Any]]
@@ -328,6 +332,7 @@ def create(
             "api/projects/",
             method="POST",
             json={
+                "annotator_evaluation_enabled": annotator_evaluation_enabled,
                 "color": color,
                 "control_weights": control_weights,
                 "created_by": convert_and_respect_annotation_metadata(
@@ -583,8 +588,10 @@ def update(
         agreement_threshold: typing.Optional[str] = OMIT,
         annotation_limit_count: typing.Optional[int] = OMIT,
         annotation_limit_percent: typing.Optional[str] = OMIT,
+        annotator_evaluation_enabled: typing.Optional[bool] = OMIT,
         annotator_evaluation_minimum_score: typing.Optional[str] = OMIT,
         annotator_evaluation_minimum_tasks: typing.Optional[int] = OMIT,
+        annotator_evaluation_onboarding_tasks: typing.Optional[int] = OMIT,
         assignment_settings: typing.Optional[AssignmentSettingsRequest] = OMIT,
         color: typing.Optional[str] = OMIT,
         comment_classification_config: typing.Optional[str] = OMIT,
@@ -642,10 +649,15 @@ def update(
 
         annotation_limit_percent : typing.Optional[str]
 
+        annotator_evaluation_enabled : typing.Optional[bool]
+            Enable annotator evaluation for the project
+
         annotator_evaluation_minimum_score : typing.Optional[str]
 
         annotator_evaluation_minimum_tasks : typing.Optional[int]
 
+        annotator_evaluation_onboarding_tasks : typing.Optional[int]
+
         assignment_settings : typing.Optional[AssignmentSettingsRequest]
 
         color : typing.Optional[str]
@@ -775,8 +787,10 @@ def update(
                 "agreement_threshold": agreement_threshold,
                 "annotation_limit_count": annotation_limit_count,
                 "annotation_limit_percent": annotation_limit_percent,
+                "annotator_evaluation_enabled": annotator_evaluation_enabled,
                 "annotator_evaluation_minimum_score": annotator_evaluation_minimum_score,
                 "annotator_evaluation_minimum_tasks": annotator_evaluation_minimum_tasks,
+                "annotator_evaluation_onboarding_tasks": annotator_evaluation_onboarding_tasks,
                 "assignment_settings": convert_and_respect_annotation_metadata(
                     object_=assignment_settings, annotation=AssignmentSettingsRequest, direction="write"
                 ),
@@ -1398,6 +1412,7 @@ async def main() -> None:
     async def create(
         self,
         *,
+        annotator_evaluation_enabled: typing.Optional[bool] = OMIT,
         color: typing.Optional[str] = OMIT,
         control_weights: typing.Optional[typing.Optional[typing.Any]] = OMIT,
         created_by: typing.Optional[UserSimpleRequest] = OMIT,
@@ -1434,6 +1449,9 @@ async def create(
 
         Parameters
         ----------
+        annotator_evaluation_enabled : typing.Optional[bool]
+            Enable annotator evaluation for the project
+
         color : typing.Optional[str]
 
         control_weights : typing.Optional[typing.Optional[typing.Any]]
@@ -1543,6 +1561,7 @@ async def main() -> None:
             "api/projects/",
             method="POST",
             json={
+                "annotator_evaluation_enabled": annotator_evaluation_enabled,
                 "color": color,
                 "control_weights": control_weights,
                 "created_by": convert_and_respect_annotation_metadata(
@@ -1822,8 +1841,10 @@ async def update(
         agreement_threshold: typing.Optional[str] = OMIT,
         annotation_limit_count: typing.Optional[int] = OMIT,
         annotation_limit_percent: typing.Optional[str] = OMIT,
+        annotator_evaluation_enabled: typing.Optional[bool] = OMIT,
         annotator_evaluation_minimum_score: typing.Optional[str] = OMIT,
         annotator_evaluation_minimum_tasks: typing.Optional[int] = OMIT,
+        annotator_evaluation_onboarding_tasks: typing.Optional[int] = OMIT,
         assignment_settings: typing.Optional[AssignmentSettingsRequest] = OMIT,
         color: typing.Optional[str] = OMIT,
         comment_classification_config: typing.Optional[str] = OMIT,
@@ -1881,10 +1902,15 @@ async def update(
 
         annotation_limit_percent : typing.Optional[str]
 
+        annotator_evaluation_enabled : typing.Optional[bool]
+            Enable annotator evaluation for the project
+
         annotator_evaluation_minimum_score : typing.Optional[str]
 
         annotator_evaluation_minimum_tasks : typing.Optional[int]
 
+        annotator_evaluation_onboarding_tasks : typing.Optional[int]
+
         assignment_settings : typing.Optional[AssignmentSettingsRequest]
 
         color : typing.Optional[str]
@@ -2022,8 +2048,10 @@ async def main() -> None:
                 "agreement_threshold": agreement_threshold,
                 "annotation_limit_count": annotation_limit_count,
                 "annotation_limit_percent": annotation_limit_percent,
+                "annotator_evaluation_enabled": annotator_evaluation_enabled,
                 "annotator_evaluation_minimum_score": annotator_evaluation_minimum_score,
                 "annotator_evaluation_minimum_tasks": annotator_evaluation_minimum_tasks,
+                "annotator_evaluation_onboarding_tasks": annotator_evaluation_onboarding_tasks,
                 "assignment_settings": convert_and_respect_annotation_metadata(
                     object_=assignment_settings, annotation=AssignmentSettingsRequest, direction="write"
                 ),

src/label_studio_sdk/projects/metrics/__init__.py

@@ -1,6 +1,6 @@
 # This file was auto-generated by Fern from our API Definition.
 
 from . import custom
-from .custom import GetLambdaCustomResponse
+from .custom import GetLambdaCustomResponse, GetLambdaCustomResponseStatus
 
-__all__ = ["GetLambdaCustomResponse", "custom"]
+__all__ = ["GetLambdaCustomResponse", "GetLambdaCustomResponseStatus", "custom"]

src/label_studio_sdk/projects/metrics/custom/__init__.py

@@ -1,5 +1,5 @@
 # This file was auto-generated by Fern from our API Definition.
 
-from .types import GetLambdaCustomResponse
+from .types import GetLambdaCustomResponse, GetLambdaCustomResponseStatus
 
-__all__ = ["GetLambdaCustomResponse"]
+__all__ = ["GetLambdaCustomResponse", "GetLambdaCustomResponseStatus"]

src/label_studio_sdk/projects/metrics/custom/client.py

@@ -41,7 +41,7 @@ def get_lambda(
         Returns
         -------
         GetLambdaCustomResponse
-            Lambda code
+            Lambda code and deployment status
 
         Examples
         --------
@@ -96,10 +96,13 @@ def update_lambda(
         id : int
 
         code : str
+            The Python code for the custom metric function.
 
         region : typing.Optional[str]
+            The AWS region for the Lambda function. Uses default if not provided.
 
         role : typing.Optional[str]
+            The AWS IAM role ARN for the Lambda function. Uses default if not provided.
 
         request_options : typing.Optional[RequestOptions]
             Request-specific configuration.
@@ -308,7 +311,7 @@ async def get_lambda(
         Returns
         -------
         GetLambdaCustomResponse
-            Lambda code
+            Lambda code and deployment status
 
         Examples
         --------
@@ -371,10 +374,13 @@ async def update_lambda(
         id : int
 
         code : str
+            The Python code for the custom metric function.
 
         region : typing.Optional[str]
+            The AWS region for the Lambda function. Uses default if not provided.
 
         role : typing.Optional[str]
+            The AWS IAM role ARN for the Lambda function. Uses default if not provided.
 
         request_options : typing.Optional[RequestOptions]
             Request-specific configuration.

src/label_studio_sdk/projects/metrics/custom/types/__init__.py

@@ -1,5 +1,6 @@
 # This file was auto-generated by Fern from our API Definition.
 
 from .get_lambda_custom_response import GetLambdaCustomResponse
+from .get_lambda_custom_response_status import GetLambdaCustomResponseStatus
 
-__all__ = ["GetLambdaCustomResponse"]
+__all__ = ["GetLambdaCustomResponse", "GetLambdaCustomResponseStatus"]

src/label_studio_sdk/projects/metrics/custom/types/get_lambda_custom_response.py

@@ -1,13 +1,18 @@
 # This file was auto-generated by Fern from our API Definition.
 
 from .....core.unchecked_base_model import UncheckedBaseModel
+from .get_lambda_custom_response_status import GetLambdaCustomResponseStatus
+import pydantic
 from .....core.pydantic_utilities import IS_PYDANTIC_V2
 import typing
-import pydantic
 
 
 class GetLambdaCustomResponse(UncheckedBaseModel):
     code: str
+    status: GetLambdaCustomResponseStatus = pydantic.Field()
+    """
+    Deployment status of the Lambda function
+    """
 
     if IS_PYDANTIC_V2:
         model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2

src/label_studio_sdk/projects/metrics/custom/types/get_lambda_custom_response_status.py

@@ -0,0 +1,7 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+GetLambdaCustomResponseStatus = typing.Union[
+    typing.Literal["NotDeployed", "Active", "Pending", "Inactive", "Failed", "NotFound", "Error"], typing.Any
+]

src/label_studio_sdk/types/all_roles_project_list.py

@@ -27,8 +27,14 @@ class AllRolesProjectList(UncheckedBaseModel):
     allow_stream: typing.Optional[bool] = None
     annotation_limit_count: typing.Optional[int] = None
     annotation_limit_percent: typing.Optional[str] = None
+    annotator_evaluation_enabled: typing.Optional[bool] = pydantic.Field(default=None)
+    """
+    Enable annotator evaluation for the project
+    """
+
     annotator_evaluation_minimum_score: typing.Optional[str] = None
     annotator_evaluation_minimum_tasks: typing.Optional[int] = None
+    annotator_evaluation_onboarding_tasks: typing.Optional[int] = None
     assignment_settings: AssignmentSettings
     blueprints: typing.Optional[typing.List[BlueprintList]] = None
     color: typing.Optional[str] = None

src/label_studio_sdk/types/lse_project.py

@@ -25,8 +25,14 @@ class LseProject(UncheckedBaseModel):
 
     annotation_limit_count: typing.Optional[int] = None
     annotation_limit_percent: typing.Optional[str] = None
+    annotator_evaluation_enabled: typing.Optional[bool] = pydantic.Field(default=None)
+    """
+    Enable annotator evaluation for the project
+    """
+
     annotator_evaluation_minimum_score: typing.Optional[str] = None
     annotator_evaluation_minimum_tasks: typing.Optional[int] = None
+    annotator_evaluation_onboarding_tasks: typing.Optional[int] = None
     assignment_settings: AssignmentSettings
     color: typing.Optional[str] = None
     comment_classification_config: typing.Optional[str] = None

src/label_studio_sdk/types/lse_project_create.py

@@ -16,6 +16,11 @@ class LseProjectCreate(UncheckedBaseModel):
     make sure, that you use correct one(Project.objects.with_counts())
     """
 
+    annotator_evaluation_enabled: typing.Optional[bool] = pydantic.Field(default=None)
+    """
+    Enable annotator evaluation for the project
+    """
+
     color: typing.Optional[str] = None
     config_has_control_tags: typing.Optional[bool] = pydantic.Field(default=None)
     """

src/label_studio_sdk/types/lse_project_response.py

@@ -25,8 +25,14 @@ class LseProjectResponse(UncheckedBaseModel):
     allow_stream: typing.Optional[bool] = None
     annotation_limit_count: typing.Optional[int] = None
     annotation_limit_percent: typing.Optional[str] = None
+    annotator_evaluation_enabled: typing.Optional[bool] = pydantic.Field(default=None)
+    """
+    Enable annotator evaluation for the project
+    """
+
     annotator_evaluation_minimum_score: typing.Optional[str] = None
     annotator_evaluation_minimum_tasks: typing.Optional[int] = None
+    annotator_evaluation_onboarding_tasks: typing.Optional[int] = None
     assignment_settings: AssignmentSettings
     color: typing.Optional[str] = None
     comment_classification_config: typing.Optional[str] = None

src/label_studio_sdk/types/lse_project_update.py

@@ -25,8 +25,14 @@ class LseProjectUpdate(UncheckedBaseModel):
 
     annotation_limit_count: typing.Optional[int] = None
     annotation_limit_percent: typing.Optional[str] = None
+    annotator_evaluation_enabled: typing.Optional[bool] = pydantic.Field(default=None)
+    """
+    Enable annotator evaluation for the project
+    """
+
     annotator_evaluation_minimum_score: typing.Optional[str] = None
     annotator_evaluation_minimum_tasks: typing.Optional[int] = None
+    annotator_evaluation_onboarding_tasks: typing.Optional[int] = None
     assignment_settings: AssignmentSettings
     color: typing.Optional[str] = None
     comment_classification_config: typing.Optional[str] = None

src/label_studio_sdk/types/project.py

@@ -16,6 +16,11 @@ class Project(UncheckedBaseModel):
     make sure, that you use correct one(Project.objects.with_counts())
     """
 
+    annotator_evaluation_enabled: typing.Optional[bool] = pydantic.Field(default=None)
+    """
+    Enable annotator evaluation for the project
+    """
+
     color: typing.Optional[str] = None
     config_has_control_tags: typing.Optional[bool] = pydantic.Field(default=None)
     """