From 1540ee0e438b76472fe38357300e65de32ff88ad Mon Sep 17 00:00:00 2001 From: fern-api <115122769+fern-api[bot]@users.noreply.github.com> Date: Thu, 9 Oct 2025 13:54:44 +0000 Subject: [PATCH 1/4] SDK regeneration --- .mock/definition/dashboard.yml | 117 ------------ .mock/definition/projects/stats.yml | 66 +++++++ .mock/openapi/openapi.yaml | 176 ++++++------------ reference.md | 93 +++++++++ src/label_studio_sdk/projects/__init__.py | 4 + .../projects/stats/__init__.py | 4 + src/label_studio_sdk/projects/stats/client.py | 149 +++++++++++++++ .../projects/stats/types/__init__.py | 6 + ...ats_users_annotators_agreement_response.py | 27 +++ ...tors_agreement_response_agreement_value.py | 5 + tests/projects/test_stats.py | 10 + 11 files changed, 423 insertions(+), 234 deletions(-) create mode 100644 src/label_studio_sdk/projects/stats/types/stats_users_annotators_agreement_response.py create mode 100644 src/label_studio_sdk/projects/stats/types/stats_users_annotators_agreement_response_agreement_value.py diff --git a/.mock/definition/dashboard.yml b/.mock/definition/dashboard.yml index 98417b5ec..f9dcbd342 100644 --- a/.mock/definition/dashboard.yml +++ b/.mock/definition/dashboard.yml @@ -21,56 +21,6 @@ types: docs: Total number of tasks source: openapi: openapi/openapi.yaml - ApiProjectsDashboardMembersRetrieveResponseStatsItem: - docs: User statistics - properties: - accepted: - type: optional - docs: Number of annotations marked as "Accepted" by reviewer - finished: - type: optional - gt: - type: optional - docs: Average agreement with Ground Truth annotations - mean_time: - type: optional - docs: Average mean time spent on annotation - median_time: - type: optional - docs: Average median time spent on annotation - predictions: - type: optional - docs: Average agreement with predictions - progress: - type: optional - docs: Fraction of annotation work done so far - rejected: - type: optional - docs: Number of annotations marked as "Rejected" by reviewer - review_score: - type: optional - docs: >- - Average reviewing score, when calling with "per_label=true", returns - dictionary with labels to score breakdown - skipped: - type: optional - source: - openapi: openapi/openapi.yaml - inline: true - ApiProjectsDashboardMembersRetrieveResponse: - docs: Task creation response - properties: - similarity: - type: optional>> - docs: Consensus statistics between different users - stats: - type: optional> - docs: Number of annotated (completed) tasks - users: - type: optional>> - docs: List of users - source: - openapi: openapi/openapi.yaml service: auth: false base-path: '' @@ -102,72 +52,5 @@ service: tasks: 1 audiences: - internal - api_projects_dashboard_members_retrieve: - path: /api/projects/{id}/dashboard-members - method: GET - auth: true - docs: >- - Get dashboard for members, including similarity matrix, user statistics - and users list. - source: - openapi: openapi/openapi.yaml - path-parameters: - id: integer - display-name: Get dashboard data for members - request: - name: ApiProjectsDashboardMembersRetrieveRequest - query-parameters: - action: - type: optional - docs: >- - All data objects will be filtered by updated_at or created_at - field - end_date: - type: optional - docs: >- - End date for dashboard stats calculation. UTC timezone by default. - Use iso format (yyyy-mm-dd-hh-mm) to specify timezone. - per_label: - type: optional - docs: Per label calculation - start_date: - type: optional - docs: >- - Start date for dashboard stats calculation. UTC timezone by - default. Use iso format (yyyy-mm-dd-hh-mm) to specify timezone. - updated_by: - type: optional - docs: >- - All task objects will be filtered by updated_by field. Only tasks - that were updated by concrete user will be taken in account. - use_kappa: - type: optional - docs: Use kappa statistics for calculation - response: - docs: Successful response returns project-related annotation statistics - type: ApiProjectsDashboardMembersRetrieveResponse - availability: deprecated - examples: - - path-parameters: - id: 1 - response: - body: - similarity: - - key: value - stats: - - accepted: 1 - finished: 1 - gt: 1.1 - mean_time: 1.1 - median_time: 1.1 - predictions: 1.1 - progress: 1 - rejected: 1 - review_score: 1.1 - skipped: 1 - users: - - key: value - audiences: - - internal source: openapi: openapi/openapi.yaml diff --git a/.mock/definition/projects/stats.yml b/.mock/definition/projects/stats.yml index 600f92a50..f6e4dd5fb 100644 --- a/.mock/definition/projects/stats.yml +++ b/.mock/definition/projects/stats.yml @@ -175,6 +175,29 @@ types: - StatsTotalAgreementResponseOne source: openapi: openapi/openapi.yaml + StatsUsersAnnotatorsAgreementResponseAgreementValue: + discriminated: false + union: + - type: double + docs: >- + Average prediction agreement score for the user (0-1) when + per_label=False + - type: map + docs: >- + Average prediction agreement score per label for the user (0-1) when + per_label=True + source: + openapi: openapi/openapi.yaml + inline: true + StatsUsersAnnotatorsAgreementResponse: + properties: + agreement: + type: >- + optional> + docs: Dictionary mapping user IDs to their prediction agreement scores + source: + openapi: openapi/openapi.yaml StatsUserPredictionAgreementResponseAveragePredictionAgreementPerUser: discriminated: false union: @@ -660,6 +683,49 @@ service: key: value audiences: - public + users_annotators_agreement: + path: /api/projects/{id}/user-stats/prediction + method: GET + auth: true + docs: >- + + Label Studio Enterprise badge +

+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise) +

+ + Get prediction agreement statistics for multiple annotators within a + project. + source: + openapi: openapi/openapi.yaml + path-parameters: + id: integer + display-name: Get prediction agreement statistics for multiple annotators + request: + name: StatsUsersAnnotatorsAgreementRequest + query-parameters: + ids: + type: string + docs: >- + Comma-separated list of annotator user IDs to get prediction + agreement scores for + per_label: + type: optional + docs: Calculate agreement per label + response: + docs: Prediction agreement statistics for multiple annotators + type: StatsUsersAnnotatorsAgreementResponse + examples: + - path-parameters: + id: 1 + query-parameters: + ids: ids + response: + body: + agreement: + key: 1.1 + audiences: + - public user_prediction_agreement: path: /api/projects/{id}/user-stats/{user_pk}/prediction method: GET diff --git a/.mock/openapi/openapi.yaml b/.mock/openapi/openapi.yaml index 210803f0f..690f8a7a1 100644 --- a/.mock/openapi/openapi.yaml +++ b/.mock/openapi/openapi.yaml @@ -7461,122 +7461,6 @@ paths: - Dashboard x-fern-audiences: - internal - /api/projects/{id}/dashboard-members: - get: - deprecated: true - description: Get dashboard for members, including similarity matrix, user statistics and users list. - operationId: api_projects_dashboard_members_retrieve - parameters: - - description: All data objects will be filtered by updated_at or created_at field - in: query - name: action - schema: - type: string - - description: End date for dashboard stats calculation. UTC timezone by default. Use iso format (yyyy-mm-dd-hh-mm) to specify timezone. - in: query - name: end_date - schema: - type: string - - in: path - name: id - required: true - schema: - type: integer - - description: Per label calculation - in: query - name: per_label - schema: - type: boolean - - description: Start date for dashboard stats calculation. UTC timezone by default. Use iso format (yyyy-mm-dd-hh-mm) to specify timezone. - in: query - name: start_date - schema: - type: string - - description: All task objects will be filtered by updated_by field. Only tasks that were updated by concrete user will be taken in account. - in: query - name: updated_by - schema: - type: string - - description: Use kappa statistics for calculation - in: query - name: use_kappa - schema: - type: boolean - responses: - '200': - content: - application/json: - schema: - description: Task creation response - properties: - similarity: - description: Consensus statistics between different users - items: - title: Similarities, common tasks - type: object - type: array - stats: - description: Number of annotated (completed) tasks - items: - description: User statistics - properties: - accepted: - description: Number of annotations marked as "Accepted" by reviewer - title: Accepted tasks - type: integer - finished: - title: Finished tasks - type: integer - gt: - description: Average agreement with Ground Truth annotations - title: Ground Truth score - type: number - mean_time: - description: Average mean time spent on annotation - title: Mean time - type: number - median_time: - description: Average median time spent on annotation - title: Median time - type: number - predictions: - description: Average agreement with predictions - title: Predictions score - type: number - progress: - description: Fraction of annotation work done so far - title: Annotation progress - type: integer - rejected: - description: Number of annotations marked as "Rejected" by reviewer - title: Rejected tasks - type: integer - review_score: - description: Average reviewing score, when calling with "per_label=true", returns dictionary with labels to score breakdown - title: Review score - type: number - skipped: - title: Skipped tasks - type: integer - title: User statistics - type: object - type: array - users: - description: List of users - items: - title: User data - type: object - type: array - title: Task creation response - type: object - description: Successful response returns project-related annotation statistics - security: - - Token: [] - summary: Get dashboard data for members - tags: - - Dashboard - x-fern-audiences: - - internal /api/projects/{id}/duplicate/: post: description: |- @@ -9846,6 +9730,64 @@ paths: - projects - stats x-fern-sdk-method-name: update_stats + /api/projects/{id}/user-stats/prediction: + get: + description: |- + + Label Studio Enterprise badge +

+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise) +

+ + Get prediction agreement statistics for multiple annotators within a project. + operationId: api_projects_user_stats_prediction_retrieve + parameters: + - in: path + name: id + required: true + schema: + type: integer + - description: Comma-separated list of annotator user IDs to get prediction agreement scores for + in: query + name: ids + required: true + schema: + type: string + - description: Calculate agreement per label + in: query + name: per_label + schema: + type: boolean + responses: + '200': + content: + application/json: + schema: + properties: + agreement: + additionalProperties: + oneOf: + - description: Average prediction agreement score for the user (0-1) when per_label=False + type: number + - additionalProperties: + type: number + description: Average prediction agreement score per label for the user (0-1) when per_label=True + type: object + description: Dictionary mapping user IDs to their prediction agreement scores + type: object + type: object + description: Prediction agreement statistics for multiple annotators + security: + - Token: [] + summary: Get prediction agreement statistics for multiple annotators + tags: + - Stats + x-fern-audiences: + - public + x-fern-sdk-group-name: + - projects + - stats + x-fern-sdk-method-name: users_annotators_agreement /api/projects/{id}/user-stats/{user_pk}/prediction: get: description: |- @@ -9856,7 +9798,7 @@ paths:

Get prediction agreement statistics for a specific user within a project. - operationId: api_projects_user_stats_prediction_retrieve + operationId: api_projects_user_stats_prediction_retrieve_2 parameters: - in: path name: id diff --git a/reference.md b/reference.md index d6018ab86..7294e437c 100644 --- a/reference.md +++ b/reference.md @@ -32165,6 +32165,99 @@ client.projects.stats.update_stats( + + + + +
client.projects.stats.users_annotators_agreement(...) +
+
+ +#### 📝 Description + +
+
+ +
+
+ + + Label Studio Enterprise badge +

+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise) +

+ +Get prediction agreement statistics for multiple annotators within a project. +
+
+
+
+ +#### 🔌 Usage + +
+
+ +
+
+ +```python +from label_studio_sdk import LabelStudio + +client = LabelStudio( + api_key="YOUR_API_KEY", +) +client.projects.stats.users_annotators_agreement( + id=1, + ids="ids", +) + +``` +
+
+
+
+ +#### ⚙️ Parameters + +
+
+ +
+
+ +**id:** `int` + +
+
+ +
+
+ +**ids:** `str` — Comma-separated list of annotator user IDs to get prediction agreement scores for + +
+
+ +
+
+ +**per_label:** `typing.Optional[bool]` — Calculate agreement per label + +
+
+ +
+
+ +**request_options:** `typing.Optional[RequestOptions]` — Request-specific configuration. + +
+
+
+
+ +
diff --git a/src/label_studio_sdk/projects/__init__.py b/src/label_studio_sdk/projects/__init__.py index 7242173d3..c3d5569af 100644 --- a/src/label_studio_sdk/projects/__init__.py +++ b/src/label_studio_sdk/projects/__init__.py @@ -53,6 +53,8 @@ StatsUserReviewScoreResponse, StatsUserReviewScoreResponsePerformanceScore, StatsUserReviewScoreResponseReviewScore, + StatsUsersAnnotatorsAgreementResponse, + StatsUsersAnnotatorsAgreementResponseAgreementValue, ) __all__ = [ @@ -103,6 +105,8 @@ "StatsUserReviewScoreResponse", "StatsUserReviewScoreResponsePerformanceScore", "StatsUserReviewScoreResponseReviewScore", + "StatsUsersAnnotatorsAgreementResponse", + "StatsUsersAnnotatorsAgreementResponseAgreementValue", "assignments", "exports", "members", diff --git a/src/label_studio_sdk/projects/stats/__init__.py b/src/label_studio_sdk/projects/stats/__init__.py index 8bda68459..537c4a345 100644 --- a/src/label_studio_sdk/projects/stats/__init__.py +++ b/src/label_studio_sdk/projects/stats/__init__.py @@ -26,6 +26,8 @@ StatsUserReviewScoreResponse, StatsUserReviewScoreResponsePerformanceScore, StatsUserReviewScoreResponseReviewScore, + StatsUsersAnnotatorsAgreementResponse, + StatsUsersAnnotatorsAgreementResponseAgreementValue, ) __all__ = [ @@ -54,4 +56,6 @@ "StatsUserReviewScoreResponse", "StatsUserReviewScoreResponsePerformanceScore", "StatsUserReviewScoreResponseReviewScore", + "StatsUsersAnnotatorsAgreementResponse", + "StatsUsersAnnotatorsAgreementResponseAgreementValue", ] diff --git a/src/label_studio_sdk/projects/stats/client.py b/src/label_studio_sdk/projects/stats/client.py index f1556c847..3173c3f88 100644 --- a/src/label_studio_sdk/projects/stats/client.py +++ b/src/label_studio_sdk/projects/stats/client.py @@ -17,6 +17,7 @@ from .types.stats_finished_tasks_response import StatsFinishedTasksResponse from .types.stats_lead_time_response import StatsLeadTimeResponse from .types.stats_total_agreement_response import StatsTotalAgreementResponse +from .types.stats_users_annotators_agreement_response import StatsUsersAnnotatorsAgreementResponse from .types.stats_user_prediction_agreement_response import StatsUserPredictionAgreementResponse from .types.stats_user_review_score_response import StatsUserReviewScoreResponse from .types.stats_user_ground_truth_agreement_response import StatsUserGroundTruthAgreementResponse @@ -689,6 +690,76 @@ def update_stats( raise ApiError(status_code=_response.status_code, body=_response.text) raise ApiError(status_code=_response.status_code, body=_response_json) + def users_annotators_agreement( + self, + id: int, + *, + ids: str, + per_label: typing.Optional[bool] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> StatsUsersAnnotatorsAgreementResponse: + """ + + Label Studio Enterprise badge +

+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise) +

+ + Get prediction agreement statistics for multiple annotators within a project. + + Parameters + ---------- + id : int + + ids : str + Comma-separated list of annotator user IDs to get prediction agreement scores for + + per_label : typing.Optional[bool] + Calculate agreement per label + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + StatsUsersAnnotatorsAgreementResponse + Prediction agreement statistics for multiple annotators + + Examples + -------- + from label_studio_sdk import LabelStudio + + client = LabelStudio( + api_key="YOUR_API_KEY", + ) + client.projects.stats.users_annotators_agreement( + id=1, + ids="ids", + ) + """ + _response = self._client_wrapper.httpx_client.request( + f"api/projects/{jsonable_encoder(id)}/user-stats/prediction", + method="GET", + params={ + "ids": ids, + "per_label": per_label, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return typing.cast( + StatsUsersAnnotatorsAgreementResponse, + construct_type( + type_=StatsUsersAnnotatorsAgreementResponse, # type: ignore + object_=_response.json(), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + def user_prediction_agreement( self, id: int, @@ -1650,6 +1721,84 @@ async def main() -> None: raise ApiError(status_code=_response.status_code, body=_response.text) raise ApiError(status_code=_response.status_code, body=_response_json) + async def users_annotators_agreement( + self, + id: int, + *, + ids: str, + per_label: typing.Optional[bool] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> StatsUsersAnnotatorsAgreementResponse: + """ + + Label Studio Enterprise badge +

+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise) +

+ + Get prediction agreement statistics for multiple annotators within a project. + + Parameters + ---------- + id : int + + ids : str + Comma-separated list of annotator user IDs to get prediction agreement scores for + + per_label : typing.Optional[bool] + Calculate agreement per label + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + StatsUsersAnnotatorsAgreementResponse + Prediction agreement statistics for multiple annotators + + Examples + -------- + import asyncio + + from label_studio_sdk import AsyncLabelStudio + + client = AsyncLabelStudio( + api_key="YOUR_API_KEY", + ) + + + async def main() -> None: + await client.projects.stats.users_annotators_agreement( + id=1, + ids="ids", + ) + + + asyncio.run(main()) + """ + _response = await self._client_wrapper.httpx_client.request( + f"api/projects/{jsonable_encoder(id)}/user-stats/prediction", + method="GET", + params={ + "ids": ids, + "per_label": per_label, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return typing.cast( + StatsUsersAnnotatorsAgreementResponse, + construct_type( + type_=StatsUsersAnnotatorsAgreementResponse, # type: ignore + object_=_response.json(), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + async def user_prediction_agreement( self, id: int, diff --git a/src/label_studio_sdk/projects/stats/types/__init__.py b/src/label_studio_sdk/projects/stats/types/__init__.py index 770c7e1fb..544b040ee 100644 --- a/src/label_studio_sdk/projects/stats/types/__init__.py +++ b/src/label_studio_sdk/projects/stats/types/__init__.py @@ -27,6 +27,10 @@ from .stats_user_review_score_response import StatsUserReviewScoreResponse from .stats_user_review_score_response_performance_score import StatsUserReviewScoreResponsePerformanceScore from .stats_user_review_score_response_review_score import StatsUserReviewScoreResponseReviewScore +from .stats_users_annotators_agreement_response import StatsUsersAnnotatorsAgreementResponse +from .stats_users_annotators_agreement_response_agreement_value import ( + StatsUsersAnnotatorsAgreementResponseAgreementValue, +) __all__ = [ "StatsAgreementAnnotatorResponse", @@ -54,4 +58,6 @@ "StatsUserReviewScoreResponse", "StatsUserReviewScoreResponsePerformanceScore", "StatsUserReviewScoreResponseReviewScore", + "StatsUsersAnnotatorsAgreementResponse", + "StatsUsersAnnotatorsAgreementResponseAgreementValue", ] diff --git a/src/label_studio_sdk/projects/stats/types/stats_users_annotators_agreement_response.py b/src/label_studio_sdk/projects/stats/types/stats_users_annotators_agreement_response.py new file mode 100644 index 000000000..5f31eef93 --- /dev/null +++ b/src/label_studio_sdk/projects/stats/types/stats_users_annotators_agreement_response.py @@ -0,0 +1,27 @@ +# This file was auto-generated by Fern from our API Definition. + +from ....core.unchecked_base_model import UncheckedBaseModel +import typing +from .stats_users_annotators_agreement_response_agreement_value import ( + StatsUsersAnnotatorsAgreementResponseAgreementValue, +) +import pydantic +from ....core.pydantic_utilities import IS_PYDANTIC_V2 + + +class StatsUsersAnnotatorsAgreementResponse(UncheckedBaseModel): + agreement: typing.Optional[typing.Dict[str, StatsUsersAnnotatorsAgreementResponseAgreementValue]] = pydantic.Field( + default=None + ) + """ + Dictionary mapping user IDs to their prediction agreement scores + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/src/label_studio_sdk/projects/stats/types/stats_users_annotators_agreement_response_agreement_value.py b/src/label_studio_sdk/projects/stats/types/stats_users_annotators_agreement_response_agreement_value.py new file mode 100644 index 000000000..d971a9197 --- /dev/null +++ b/src/label_studio_sdk/projects/stats/types/stats_users_annotators_agreement_response_agreement_value.py @@ -0,0 +1,5 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +StatsUsersAnnotatorsAgreementResponseAgreementValue = typing.Union[float, typing.Dict[str, float]] diff --git a/tests/projects/test_stats.py b/tests/projects/test_stats.py index a485a8fca..8f3acc49b 100644 --- a/tests/projects/test_stats.py +++ b/tests/projects/test_stats.py @@ -144,6 +144,16 @@ async def test_update_stats(client: LabelStudio, async_client: AsyncLabelStudio) validate_response(async_response, expected_response, expected_types) +async def test_users_annotators_agreement(client: LabelStudio, async_client: AsyncLabelStudio) -> None: + expected_response: typing.Any = {"agreement": {"key": 1.1}} + expected_types: typing.Any = {"agreement": ("dict", {0: (None, None)})} + response = client.projects.stats.users_annotators_agreement(id=1, ids="ids") + validate_response(response, expected_response, expected_types) + + async_response = await async_client.projects.stats.users_annotators_agreement(id=1, ids="ids") + validate_response(async_response, expected_response, expected_types) + + async def test_user_prediction_agreement(client: LabelStudio, async_client: AsyncLabelStudio) -> None: expected_response: typing.Any = {"average_prediction_agreement_per_user": 1.1} expected_types: typing.Any = {"average_prediction_agreement_per_user": None} From 5b567ef3e362065f2713712ce79a7ae5b13d7aad Mon Sep 17 00:00:00 2001 From: "fern-api[bot]" <115122769+fern-api[bot]@users.noreply.github.com> Date: Thu, 9 Oct 2025 18:10:46 +0000 Subject: [PATCH 2/4] SDK regeneration --- .mock/definition/projects/stats.yml | 9 +++++---- .mock/openapi/openapi.yaml | 7 ++++--- reference.md | 6 +++--- src/label_studio_sdk/projects/stats/client.py | 12 ++++++------ 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.mock/definition/projects/stats.yml b/.mock/definition/projects/stats.yml index f6e4dd5fb..7a6e2b8d8 100644 --- a/.mock/definition/projects/stats.yml +++ b/.mock/definition/projects/stats.yml @@ -520,7 +520,7 @@ service: ids: type: string docs: >- - Comma-separated list of annotator user IDs to get agreement scores + Comma separated list of annotator user IDs to get agreement scores for response: docs: Multiple annotator agreement statistics @@ -707,11 +707,12 @@ service: ids: type: string docs: >- - Comma-separated list of annotator user IDs to get prediction - agreement scores for + Comma separated list of annotator user IDs to get agreement scores + for per_label: type: optional - docs: Calculate agreement per label + default: false + docs: Per label response: docs: Prediction agreement statistics for multiple annotators type: StatsUsersAnnotatorsAgreementResponse diff --git a/.mock/openapi/openapi.yaml b/.mock/openapi/openapi.yaml index 690f8a7a1..7ceef5dc4 100644 --- a/.mock/openapi/openapi.yaml +++ b/.mock/openapi/openapi.yaml @@ -8992,7 +8992,7 @@ paths: required: true schema: type: integer - - description: Comma-separated list of annotator user IDs to get agreement scores for + - description: Comma separated list of annotator user IDs to get agreement scores for in: query name: ids required: true @@ -9747,16 +9747,17 @@ paths: required: true schema: type: integer - - description: Comma-separated list of annotator user IDs to get prediction agreement scores for + - description: Comma separated list of annotator user IDs to get agreement scores for in: query name: ids required: true schema: type: string - - description: Calculate agreement per label + - description: Per label in: query name: per_label schema: + default: false type: boolean responses: '200': diff --git a/reference.md b/reference.md index 7294e437c..eea438cf4 100644 --- a/reference.md +++ b/reference.md @@ -31767,7 +31767,7 @@ client.projects.stats.agreement_annotators(
-**ids:** `str` — Comma-separated list of annotator user IDs to get agreement scores for +**ids:** `str` — Comma separated list of annotator user IDs to get agreement scores for
@@ -32234,7 +32234,7 @@ client.projects.stats.users_annotators_agreement(
-**ids:** `str` — Comma-separated list of annotator user IDs to get prediction agreement scores for +**ids:** `str` — Comma separated list of annotator user IDs to get agreement scores for
@@ -32242,7 +32242,7 @@ client.projects.stats.users_annotators_agreement(
-**per_label:** `typing.Optional[bool]` — Calculate agreement per label +**per_label:** `typing.Optional[bool]` — Per label
diff --git a/src/label_studio_sdk/projects/stats/client.py b/src/label_studio_sdk/projects/stats/client.py index 3173c3f88..99269e60c 100644 --- a/src/label_studio_sdk/projects/stats/client.py +++ b/src/label_studio_sdk/projects/stats/client.py @@ -374,7 +374,7 @@ def agreement_annotators( id : int ids : str - Comma-separated list of annotator user IDs to get agreement scores for + Comma separated list of annotator user IDs to get agreement scores for request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -712,10 +712,10 @@ def users_annotators_agreement( id : int ids : str - Comma-separated list of annotator user IDs to get prediction agreement scores for + Comma separated list of annotator user IDs to get agreement scores for per_label : typing.Optional[bool] - Calculate agreement per label + Per label request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -1355,7 +1355,7 @@ async def agreement_annotators( id : int ids : str - Comma-separated list of annotator user IDs to get agreement scores for + Comma separated list of annotator user IDs to get agreement scores for request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -1743,10 +1743,10 @@ async def users_annotators_agreement( id : int ids : str - Comma-separated list of annotator user IDs to get prediction agreement scores for + Comma separated list of annotator user IDs to get agreement scores for per_label : typing.Optional[bool] - Calculate agreement per label + Per label request_options : typing.Optional[RequestOptions] Request-specific configuration. From 95375314182424b2a71403fe051115de4c0dd37c Mon Sep 17 00:00:00 2001 From: "fern-api[bot]" <115122769+fern-api[bot]@users.noreply.github.com> Date: Thu, 9 Oct 2025 19:12:26 +0000 Subject: [PATCH 3/4] SDK regeneration --- .mock/definition/projects/stats.yml | 12 ++++----- .mock/openapi/openapi.yaml | 2 +- reference.md | 4 +-- src/label_studio_sdk/projects/__init__.py | 8 +++--- .../projects/stats/__init__.py | 8 +++--- src/label_studio_sdk/projects/stats/client.py | 26 +++++++++---------- .../projects/stats/types/__init__.py | 10 +++---- ...ts_users_prediction_agreement_response.py} | 8 +++--- ...ion_agreement_response_agreement_value.py} | 2 +- tests/projects/test_stats.py | 6 ++--- 10 files changed, 43 insertions(+), 43 deletions(-) rename src/label_studio_sdk/projects/stats/types/{stats_users_annotators_agreement_response.py => stats_users_prediction_agreement_response.py} (67%) rename src/label_studio_sdk/projects/stats/types/{stats_users_annotators_agreement_response_agreement_value.py => stats_users_prediction_agreement_response_agreement_value.py} (64%) diff --git a/.mock/definition/projects/stats.yml b/.mock/definition/projects/stats.yml index 7a6e2b8d8..e9c628440 100644 --- a/.mock/definition/projects/stats.yml +++ b/.mock/definition/projects/stats.yml @@ -175,7 +175,7 @@ types: - StatsTotalAgreementResponseOne source: openapi: openapi/openapi.yaml - StatsUsersAnnotatorsAgreementResponseAgreementValue: + StatsUsersPredictionAgreementResponseAgreementValue: discriminated: false union: - type: double @@ -189,12 +189,12 @@ types: source: openapi: openapi/openapi.yaml inline: true - StatsUsersAnnotatorsAgreementResponse: + StatsUsersPredictionAgreementResponse: properties: agreement: type: >- optional> + StatsUsersPredictionAgreementResponseAgreementValue>> docs: Dictionary mapping user IDs to their prediction agreement scores source: openapi: openapi/openapi.yaml @@ -683,7 +683,7 @@ service: key: value audiences: - public - users_annotators_agreement: + users_prediction_agreement: path: /api/projects/{id}/user-stats/prediction method: GET auth: true @@ -702,7 +702,7 @@ service: id: integer display-name: Get prediction agreement statistics for multiple annotators request: - name: StatsUsersAnnotatorsAgreementRequest + name: StatsUsersPredictionAgreementRequest query-parameters: ids: type: string @@ -715,7 +715,7 @@ service: docs: Per label response: docs: Prediction agreement statistics for multiple annotators - type: StatsUsersAnnotatorsAgreementResponse + type: StatsUsersPredictionAgreementResponse examples: - path-parameters: id: 1 diff --git a/.mock/openapi/openapi.yaml b/.mock/openapi/openapi.yaml index 7ceef5dc4..09c2bff02 100644 --- a/.mock/openapi/openapi.yaml +++ b/.mock/openapi/openapi.yaml @@ -9788,7 +9788,7 @@ paths: x-fern-sdk-group-name: - projects - stats - x-fern-sdk-method-name: users_annotators_agreement + x-fern-sdk-method-name: users_prediction_agreement /api/projects/{id}/user-stats/{user_pk}/prediction: get: description: |- diff --git a/reference.md b/reference.md index eea438cf4..bb00daf16 100644 --- a/reference.md +++ b/reference.md @@ -32169,7 +32169,7 @@ client.projects.stats.update_stats( -
client.projects.stats.users_annotators_agreement(...) +
client.projects.stats.users_prediction_agreement(...)
@@ -32207,7 +32207,7 @@ from label_studio_sdk import LabelStudio client = LabelStudio( api_key="YOUR_API_KEY", ) -client.projects.stats.users_annotators_agreement( +client.projects.stats.users_prediction_agreement( id=1, ids="ids", ) diff --git a/src/label_studio_sdk/projects/__init__.py b/src/label_studio_sdk/projects/__init__.py index c3d5569af..5f563c086 100644 --- a/src/label_studio_sdk/projects/__init__.py +++ b/src/label_studio_sdk/projects/__init__.py @@ -53,8 +53,8 @@ StatsUserReviewScoreResponse, StatsUserReviewScoreResponsePerformanceScore, StatsUserReviewScoreResponseReviewScore, - StatsUsersAnnotatorsAgreementResponse, - StatsUsersAnnotatorsAgreementResponseAgreementValue, + StatsUsersPredictionAgreementResponse, + StatsUsersPredictionAgreementResponseAgreementValue, ) __all__ = [ @@ -105,8 +105,8 @@ "StatsUserReviewScoreResponse", "StatsUserReviewScoreResponsePerformanceScore", "StatsUserReviewScoreResponseReviewScore", - "StatsUsersAnnotatorsAgreementResponse", - "StatsUsersAnnotatorsAgreementResponseAgreementValue", + "StatsUsersPredictionAgreementResponse", + "StatsUsersPredictionAgreementResponseAgreementValue", "assignments", "exports", "members", diff --git a/src/label_studio_sdk/projects/stats/__init__.py b/src/label_studio_sdk/projects/stats/__init__.py index 537c4a345..9ec1b022b 100644 --- a/src/label_studio_sdk/projects/stats/__init__.py +++ b/src/label_studio_sdk/projects/stats/__init__.py @@ -26,8 +26,8 @@ StatsUserReviewScoreResponse, StatsUserReviewScoreResponsePerformanceScore, StatsUserReviewScoreResponseReviewScore, - StatsUsersAnnotatorsAgreementResponse, - StatsUsersAnnotatorsAgreementResponseAgreementValue, + StatsUsersPredictionAgreementResponse, + StatsUsersPredictionAgreementResponseAgreementValue, ) __all__ = [ @@ -56,6 +56,6 @@ "StatsUserReviewScoreResponse", "StatsUserReviewScoreResponsePerformanceScore", "StatsUserReviewScoreResponseReviewScore", - "StatsUsersAnnotatorsAgreementResponse", - "StatsUsersAnnotatorsAgreementResponseAgreementValue", + "StatsUsersPredictionAgreementResponse", + "StatsUsersPredictionAgreementResponseAgreementValue", ] diff --git a/src/label_studio_sdk/projects/stats/client.py b/src/label_studio_sdk/projects/stats/client.py index 99269e60c..cd1beecff 100644 --- a/src/label_studio_sdk/projects/stats/client.py +++ b/src/label_studio_sdk/projects/stats/client.py @@ -17,7 +17,7 @@ from .types.stats_finished_tasks_response import StatsFinishedTasksResponse from .types.stats_lead_time_response import StatsLeadTimeResponse from .types.stats_total_agreement_response import StatsTotalAgreementResponse -from .types.stats_users_annotators_agreement_response import StatsUsersAnnotatorsAgreementResponse +from .types.stats_users_prediction_agreement_response import StatsUsersPredictionAgreementResponse from .types.stats_user_prediction_agreement_response import StatsUserPredictionAgreementResponse from .types.stats_user_review_score_response import StatsUserReviewScoreResponse from .types.stats_user_ground_truth_agreement_response import StatsUserGroundTruthAgreementResponse @@ -690,14 +690,14 @@ def update_stats( raise ApiError(status_code=_response.status_code, body=_response.text) raise ApiError(status_code=_response.status_code, body=_response_json) - def users_annotators_agreement( + def users_prediction_agreement( self, id: int, *, ids: str, per_label: typing.Optional[bool] = None, request_options: typing.Optional[RequestOptions] = None, - ) -> StatsUsersAnnotatorsAgreementResponse: + ) -> StatsUsersPredictionAgreementResponse: """ Label Studio Enterprise badge @@ -722,7 +722,7 @@ def users_annotators_agreement( Returns ------- - StatsUsersAnnotatorsAgreementResponse + StatsUsersPredictionAgreementResponse Prediction agreement statistics for multiple annotators Examples @@ -732,7 +732,7 @@ def users_annotators_agreement( client = LabelStudio( api_key="YOUR_API_KEY", ) - client.projects.stats.users_annotators_agreement( + client.projects.stats.users_prediction_agreement( id=1, ids="ids", ) @@ -749,9 +749,9 @@ def users_annotators_agreement( try: if 200 <= _response.status_code < 300: return typing.cast( - StatsUsersAnnotatorsAgreementResponse, + StatsUsersPredictionAgreementResponse, construct_type( - type_=StatsUsersAnnotatorsAgreementResponse, # type: ignore + type_=StatsUsersPredictionAgreementResponse, # type: ignore object_=_response.json(), ), ) @@ -1721,14 +1721,14 @@ async def main() -> None: raise ApiError(status_code=_response.status_code, body=_response.text) raise ApiError(status_code=_response.status_code, body=_response_json) - async def users_annotators_agreement( + async def users_prediction_agreement( self, id: int, *, ids: str, per_label: typing.Optional[bool] = None, request_options: typing.Optional[RequestOptions] = None, - ) -> StatsUsersAnnotatorsAgreementResponse: + ) -> StatsUsersPredictionAgreementResponse: """ Label Studio Enterprise badge @@ -1753,7 +1753,7 @@ async def users_annotators_agreement( Returns ------- - StatsUsersAnnotatorsAgreementResponse + StatsUsersPredictionAgreementResponse Prediction agreement statistics for multiple annotators Examples @@ -1768,7 +1768,7 @@ async def users_annotators_agreement( async def main() -> None: - await client.projects.stats.users_annotators_agreement( + await client.projects.stats.users_prediction_agreement( id=1, ids="ids", ) @@ -1788,9 +1788,9 @@ async def main() -> None: try: if 200 <= _response.status_code < 300: return typing.cast( - StatsUsersAnnotatorsAgreementResponse, + StatsUsersPredictionAgreementResponse, construct_type( - type_=StatsUsersAnnotatorsAgreementResponse, # type: ignore + type_=StatsUsersPredictionAgreementResponse, # type: ignore object_=_response.json(), ), ) diff --git a/src/label_studio_sdk/projects/stats/types/__init__.py b/src/label_studio_sdk/projects/stats/types/__init__.py index 544b040ee..880b95f3f 100644 --- a/src/label_studio_sdk/projects/stats/types/__init__.py +++ b/src/label_studio_sdk/projects/stats/types/__init__.py @@ -27,9 +27,9 @@ from .stats_user_review_score_response import StatsUserReviewScoreResponse from .stats_user_review_score_response_performance_score import StatsUserReviewScoreResponsePerformanceScore from .stats_user_review_score_response_review_score import StatsUserReviewScoreResponseReviewScore -from .stats_users_annotators_agreement_response import StatsUsersAnnotatorsAgreementResponse -from .stats_users_annotators_agreement_response_agreement_value import ( - StatsUsersAnnotatorsAgreementResponseAgreementValue, +from .stats_users_prediction_agreement_response import StatsUsersPredictionAgreementResponse +from .stats_users_prediction_agreement_response_agreement_value import ( + StatsUsersPredictionAgreementResponseAgreementValue, ) __all__ = [ @@ -58,6 +58,6 @@ "StatsUserReviewScoreResponse", "StatsUserReviewScoreResponsePerformanceScore", "StatsUserReviewScoreResponseReviewScore", - "StatsUsersAnnotatorsAgreementResponse", - "StatsUsersAnnotatorsAgreementResponseAgreementValue", + "StatsUsersPredictionAgreementResponse", + "StatsUsersPredictionAgreementResponseAgreementValue", ] diff --git a/src/label_studio_sdk/projects/stats/types/stats_users_annotators_agreement_response.py b/src/label_studio_sdk/projects/stats/types/stats_users_prediction_agreement_response.py similarity index 67% rename from src/label_studio_sdk/projects/stats/types/stats_users_annotators_agreement_response.py rename to src/label_studio_sdk/projects/stats/types/stats_users_prediction_agreement_response.py index 5f31eef93..bc431ddeb 100644 --- a/src/label_studio_sdk/projects/stats/types/stats_users_annotators_agreement_response.py +++ b/src/label_studio_sdk/projects/stats/types/stats_users_prediction_agreement_response.py @@ -2,15 +2,15 @@ from ....core.unchecked_base_model import UncheckedBaseModel import typing -from .stats_users_annotators_agreement_response_agreement_value import ( - StatsUsersAnnotatorsAgreementResponseAgreementValue, +from .stats_users_prediction_agreement_response_agreement_value import ( + StatsUsersPredictionAgreementResponseAgreementValue, ) import pydantic from ....core.pydantic_utilities import IS_PYDANTIC_V2 -class StatsUsersAnnotatorsAgreementResponse(UncheckedBaseModel): - agreement: typing.Optional[typing.Dict[str, StatsUsersAnnotatorsAgreementResponseAgreementValue]] = pydantic.Field( +class StatsUsersPredictionAgreementResponse(UncheckedBaseModel): + agreement: typing.Optional[typing.Dict[str, StatsUsersPredictionAgreementResponseAgreementValue]] = pydantic.Field( default=None ) """ diff --git a/src/label_studio_sdk/projects/stats/types/stats_users_annotators_agreement_response_agreement_value.py b/src/label_studio_sdk/projects/stats/types/stats_users_prediction_agreement_response_agreement_value.py similarity index 64% rename from src/label_studio_sdk/projects/stats/types/stats_users_annotators_agreement_response_agreement_value.py rename to src/label_studio_sdk/projects/stats/types/stats_users_prediction_agreement_response_agreement_value.py index d971a9197..7bc134f98 100644 --- a/src/label_studio_sdk/projects/stats/types/stats_users_annotators_agreement_response_agreement_value.py +++ b/src/label_studio_sdk/projects/stats/types/stats_users_prediction_agreement_response_agreement_value.py @@ -2,4 +2,4 @@ import typing -StatsUsersAnnotatorsAgreementResponseAgreementValue = typing.Union[float, typing.Dict[str, float]] +StatsUsersPredictionAgreementResponseAgreementValue = typing.Union[float, typing.Dict[str, float]] diff --git a/tests/projects/test_stats.py b/tests/projects/test_stats.py index 8f3acc49b..933fc18ea 100644 --- a/tests/projects/test_stats.py +++ b/tests/projects/test_stats.py @@ -144,13 +144,13 @@ async def test_update_stats(client: LabelStudio, async_client: AsyncLabelStudio) validate_response(async_response, expected_response, expected_types) -async def test_users_annotators_agreement(client: LabelStudio, async_client: AsyncLabelStudio) -> None: +async def test_users_prediction_agreement(client: LabelStudio, async_client: AsyncLabelStudio) -> None: expected_response: typing.Any = {"agreement": {"key": 1.1}} expected_types: typing.Any = {"agreement": ("dict", {0: (None, None)})} - response = client.projects.stats.users_annotators_agreement(id=1, ids="ids") + response = client.projects.stats.users_prediction_agreement(id=1, ids="ids") validate_response(response, expected_response, expected_types) - async_response = await async_client.projects.stats.users_annotators_agreement(id=1, ids="ids") + async_response = await async_client.projects.stats.users_prediction_agreement(id=1, ids="ids") validate_response(async_response, expected_response, expected_types) From 8a328fe1b3d9d50154a69b6b23668e64c5aa86ec Mon Sep 17 00:00:00 2001 From: "fern-api[bot]" <115122769+fern-api[bot]@users.noreply.github.com> Date: Thu, 9 Oct 2025 21:27:51 +0000 Subject: [PATCH 4/4] SDK regeneration