diff --git a/label_studio/data_import/api.py b/label_studio/data_import/api.py index 65d428bc049a..8aef296d8d48 100644 --- a/label_studio/data_import/api.py +++ b/label_studio/data_import/api.py @@ -19,7 +19,6 @@ from django.utils.decorators import method_decorator from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema -from label_studio_sdk.label_interface import LabelInterface from projects.models import Project, ProjectImport, ProjectReimport from ranged_fileresponse import RangedFileResponse from rest_framework import generics, status @@ -267,31 +266,7 @@ def sync_import(self, request, project, preannotated_from_fields, commit_to_proj if preannotated_from_fields: # turn flat task JSONs {"column1": value, "column2": value} into {"data": {"column1"..}, "predictions": [{..."column2"}] - parsed_data = reformat_predictions(parsed_data, preannotated_from_fields, project) - - # Conditionally validate predictions: skip when label config is default during project creation - if project.label_config_is_not_default: - validation_errors = [] - li = LabelInterface(project.label_config) - - for i, task in enumerate(parsed_data): - if 'predictions' in task: - for j, prediction in enumerate(task['predictions']): - try: - validation_errors_list = li.validate_prediction(prediction, return_errors=True) - if validation_errors_list: - for error in validation_errors_list: - validation_errors.append(f'Task {i}, prediction {j}: {error}') - except Exception as e: - error_msg = f'Task {i}, prediction {j}: Error validating prediction - {str(e)}' - validation_errors.append(error_msg) - - if validation_errors: - error_message = f'Prediction validation failed ({len(validation_errors)} errors):\n' - for error in validation_errors: - error_message += f'- {error}\n' - - raise ValidationError({'predictions': [error_message]}) + parsed_data = reformat_predictions(parsed_data, preannotated_from_fields) if commit_to_project: # Immediately create project tasks and update project states and counters @@ -509,55 +484,22 @@ def _create_legacy(self, project): logger.debug( f'Importing {len(self.request.data)} predictions to project {project} with {len(tasks_ids)} tasks (legacy mode)' ) - - li = LabelInterface(project.label_config) - - # Validate all predictions before creating any - validation_errors = [] predictions = [] - - for i, item in enumerate(self.request.data): - # Validate task ID + for item in self.request.data: if item.get('task') not in tasks_ids: - validation_errors.append( - f'Prediction {i}: Invalid task ID {item.get("task")} - task not found in project' + raise ValidationError( + f'{item} contains invalid "task" field: corresponding task ID couldn\'t be retrieved ' + f'from project {project} tasks' ) - continue - - # Validate prediction using LabelInterface only - try: - validation_errors_list = li.validate_prediction(item, return_errors=True) - - # If prediction is invalid, add error to validation_errors list and continue to next prediction - if validation_errors_list: - # Format errors for better readability - for error in validation_errors_list: - validation_errors.append(f'Prediction {i}: {error}') - continue - - except Exception as e: - validation_errors.append(f'Prediction {i}: Error validating prediction - {str(e)}') - continue - - # If prediction is valid, add it to predictions list to be created - try: - predictions.append( - Prediction( - task_id=item['task'], - project_id=project.id, - result=Prediction.prepare_prediction_result(item.get('result'), project), - score=item.get('score'), - model_version=item.get('model_version', 'undefined'), - ) + predictions.append( + Prediction( + task_id=item['task'], + project_id=project.id, + result=Prediction.prepare_prediction_result(item.get('result'), project), + score=item.get('score'), + model_version=item.get('model_version', 'undefined'), ) - except Exception as e: - validation_errors.append(f'Prediction {i}: Failed to create prediction - {str(e)}') - continue - - # If there are validation errors, raise them before creating any predictions - if validation_errors: - raise ValidationError(validation_errors) - + ) predictions_obj = Prediction.objects.bulk_create(predictions, batch_size=settings.BATCH_SIZE) start_job_async_or_sync(update_tasks_counters, Task.objects.filter(id__in=tasks_ids)) return Response({'created': len(predictions_obj)}, status=status.HTTP_201_CREATED) diff --git a/label_studio/data_import/functions.py b/label_studio/data_import/functions.py index c88b969d033c..bfee2d13ae05 100644 --- a/label_studio/data_import/functions.py +++ b/label_studio/data_import/functions.py @@ -7,9 +7,7 @@ from core.utils.common import load_func from django.conf import settings from django.db import transaction -from label_studio_sdk.label_interface import LabelInterface from projects.models import ProjectImport, ProjectReimport, ProjectSummary -from rest_framework.exceptions import ValidationError from tasks.models import Task from users.models import User from webhooks.models import WebhookAction @@ -48,35 +46,7 @@ def async_import_background( if project_import.preannotated_from_fields: # turn flat task JSONs {"column1": value, "column2": value} into {"data": {"column1"..}, "predictions": [{..."column2"}] - tasks = reformat_predictions(tasks, project_import.preannotated_from_fields, project) - - # Always validate predictions regardless of commit_to_project setting - if project.label_config_is_not_default: - validation_errors = [] - li = LabelInterface(project.label_config) - - for i, task in enumerate(tasks): - if 'predictions' in task: - for j, prediction in enumerate(task['predictions']): - try: - validation_errors_list = li.validate_prediction(prediction, return_errors=True) - if validation_errors_list: - for error in validation_errors_list: - validation_errors.append(f'Task {i}, prediction {j}: {error}') - except Exception as e: - error_msg = f'Task {i}, prediction {j}: Error validating prediction - {str(e)}' - validation_errors.append(error_msg) - logger.error(f'Exception during validation: {error_msg}') - - if validation_errors: - error_message = f'Prediction validation failed ({len(validation_errors)} errors):\n' - for error in validation_errors: - error_message += f'- {error}\n' - - project_import.error = error_message - project_import.status = ProjectImport.Status.FAILED - project_import.save() - return + tasks = reformat_predictions(tasks, project_import.preannotated_from_fields) if project_import.commit_to_project: with transaction.atomic(): @@ -86,41 +56,32 @@ def async_import_background( # Immediately create project tasks and update project states and counters serializer = ImportApiSerializer(data=tasks, many=True, context={'project': project}) serializer.is_valid(raise_exception=True) + tasks = serializer.save(project_id=project.id) + emit_webhooks_for_instance(user.active_organization, project, WebhookAction.TASKS_CREATED, tasks) - try: - tasks = serializer.save(project_id=project.id) - emit_webhooks_for_instance(user.active_organization, project, WebhookAction.TASKS_CREATED, tasks) - - task_count = len(tasks) - annotation_count = len(serializer.db_annotations) - prediction_count = len(serializer.db_predictions) - # Update counters (like total_annotations) for new tasks and after bulk update tasks stats. It should be a - # single operation as counters affect bulk is_labeled update - - recalculate_stats_counts = { - 'task_count': task_count, - 'annotation_count': annotation_count, - 'prediction_count': prediction_count, - } - - project.update_tasks_counters_and_task_states( - tasks_queryset=tasks, - maximum_annotations_changed=False, - overlap_cohort_percentage_changed=False, - tasks_number_changed=True, - recalculate_stats_counts=recalculate_stats_counts, - ) - logger.info('Tasks bulk_update finished (async import)') - - summary.update_data_columns(tasks) - # TODO: summary.update_created_annotations_and_labels - except Exception as e: - # Handle any other unexpected errors during task creation - error_message = f'Error creating tasks: {str(e)}' - project_import.error = error_message - project_import.status = ProjectImport.Status.FAILED - project_import.save() - return + task_count = len(tasks) + annotation_count = len(serializer.db_annotations) + prediction_count = len(serializer.db_predictions) + # Update counters (like total_annotations) for new tasks and after bulk update tasks stats. It should be a + # single operation as counters affect bulk is_labeled update + + recalculate_stats_counts = { + 'task_count': task_count, + 'annotation_count': annotation_count, + 'prediction_count': prediction_count, + } + + project.update_tasks_counters_and_task_states( + tasks_queryset=tasks, + maximum_annotations_changed=False, + overlap_cohort_percentage_changed=False, + tasks_number_changed=True, + recalculate_stats_counts=recalculate_stats_counts, + ) + logger.info('Tasks bulk_update finished (async import)') + + summary.update_data_columns(tasks) + # TODO: summary.update_created_annotations_and_labels else: # Do nothing - just output file upload ids for further use task_count = len(tasks) @@ -159,103 +120,13 @@ def set_reimport_background_failure(job, connection, type, value, _): ) -def reformat_predictions(tasks, preannotated_from_fields, project=None): - """ - Transform flat task JSON objects into proper format with separate data and predictions fields. - Also validates the predictions to ensure they are properly formatted using LabelInterface. - - Args: - tasks: List of task data - preannotated_from_fields: List of field names to convert to predictions - project: Optional project instance to determine correct to_name and type from label config - """ +def reformat_predictions(tasks, preannotated_from_fields): new_tasks = [] - validation_errors = [] - - # If project is provided, create LabelInterface to determine correct mappings - li = None - if project: - try: - li = LabelInterface(project.label_config) - except Exception as e: - logger.warning(f'Could not create LabelInterface for project {project.id}: {e}') - - for task_index, task in enumerate(tasks): + for task in tasks: if 'data' in task: - task_data = task['data'] - else: - task_data = task - - predictions = [] - for field in preannotated_from_fields: - if field not in task_data: - validation_errors.append(f"Task {task_index}: Preannotated field '{field}' not found in task data") - continue - - value = task_data[field] - if value is not None: - # Try to determine correct to_name and type from project configuration - to_name = 'text' # Default fallback - prediction_type = 'choices' # Default fallback - - if li: - # Find a control tag that matches the field name - try: - control_tag = li.get_control(field) - # Use the control's to_name and determine type - if hasattr(control_tag, 'to_name') and control_tag.to_name: - to_name = ( - control_tag.to_name[0] - if isinstance(control_tag.to_name, list) - else control_tag.to_name - ) - prediction_type = control_tag.tag.lower() - except Exception: - # Control not found, use defaults - pass - - # Create prediction from preannotated field - # Handle different types of values - if isinstance(value, dict): - # For complex structures like bounding boxes, use the value directly - prediction_value = value - else: - # For simple values, use the prediction_type as the key - # Handle cases where the type doesn't match the expected key - value_key = prediction_type - if prediction_type == 'textarea': - value_key = 'text' - - # Most types expect lists, but some expect single values - if prediction_type in ['rating', 'number', 'datetime']: - prediction_value = {value_key: value} - else: - # Wrap in list for most types - prediction_value = {value_key: [value] if not isinstance(value, list) else value} - - prediction = { - 'result': [ - { - 'from_name': field, - 'to_name': to_name, - 'type': prediction_type, - 'value': prediction_value, - } - ], - 'score': 1.0, - 'model_version': 'preannotated', - } - - predictions.append(prediction) - - # Create new task structure - new_task = {'data': task_data, 'predictions': predictions} - new_tasks.append(new_task) - - # If there are validation errors, raise them - if validation_errors: - raise ValidationError({'preannotated_fields': validation_errors}) - + task = task['data'] + predictions = [{'result': task.pop(field)} for field in preannotated_from_fields] + new_tasks.append({'data': task, 'predictions': predictions}) return new_tasks diff --git a/label_studio/io_storages/base_models.py b/label_studio/io_storages/base_models.py index 31a5dd44b8f2..416851d8a14c 100644 --- a/label_studio/io_storages/base_models.py +++ b/label_studio/io_storages/base_models.py @@ -6,7 +6,6 @@ import json import logging import os -import sys import traceback as tb from concurrent.futures import ThreadPoolExecutor from dataclasses import asdict @@ -30,7 +29,6 @@ from django.utils.translation import gettext_lazy as _ from django_rq import job from io_storages.utils import StorageObject, get_uri_via_regex, parse_bucket_uri -from rest_framework.exceptions import ValidationError from rq.job import Job from tasks.models import Annotation, Task from tasks.serializers import AnnotationSerializer, PredictionSerializer @@ -54,7 +52,6 @@ class Status(models.TextChoices): IN_PROGRESS = 'in_progress', _('In progress') FAILED = 'failed', _('Failed') COMPLETED = 'completed', _('Completed') - COMPLETED_WITH_ERRORS = 'completed_with_errors', _('Completed with errors') class Meta: abstract = True @@ -147,56 +144,9 @@ def info_set_completed(self, last_sync_count, **kwargs): self.meta.update(kwargs) self.save(update_fields=['status', 'meta', 'last_sync', 'last_sync_count']) - def info_set_completed_with_errors(self, last_sync_count, validation_errors, **kwargs): - self.status = self.Status.COMPLETED_WITH_ERRORS - self.last_sync = timezone.now() - self.last_sync_count = last_sync_count - self.traceback = '\n'.join(validation_errors) - time_completed = timezone.now() - self.meta['time_completed'] = str(time_completed) - self.meta['duration'] = (time_completed - self.time_in_progress).total_seconds() - self.meta['tasks_failed_validation'] = len(validation_errors) - self.meta.update(kwargs) - self.save(update_fields=['status', 'meta', 'last_sync', 'last_sync_count', 'traceback']) - def info_set_failed(self): self.status = self.Status.FAILED - - # Get the current exception info - exc_type, exc_value, exc_traceback = sys.exc_info() - - # Extract human-readable error messages from ValidationError - if exc_type and issubclass(exc_type, ValidationError): - error_messages = [] - if hasattr(exc_value, 'detail'): - # Handle ValidationError.detail which can be a dict or list - if isinstance(exc_value.detail, dict): - for field, errors in exc_value.detail.items(): - if isinstance(errors, list): - for error in errors: - if hasattr(error, 'string'): - error_messages.append(error.string) - else: - error_messages.append(str(error)) - else: - error_messages.append(str(errors)) - elif isinstance(exc_value.detail, list): - for error in exc_value.detail: - if hasattr(error, 'string'): - error_messages.append(error.string) - else: - error_messages.append(str(error)) - else: - error_messages.append(str(exc_value.detail)) - - # Use human-readable messages if available, otherwise fall back to full traceback - if error_messages: - self.traceback = '\n'.join(error_messages) - else: - self.traceback = str(tb.format_exc()) - else: - # For non-ValidationError exceptions, use the full traceback - self.traceback = str(tb.format_exc()) + self.traceback = str(tb.format_exc()) time_failure = timezone.now() @@ -497,9 +447,7 @@ def add_task(cls, project, maximum_annotations, max_inner_id, storage, link_obje prediction['task'] = task.id prediction['project'] = project.id prediction_ser = PredictionSerializer(data=predictions, many=True) - - # Always validate predictions and raise exception if invalid - if prediction_ser.is_valid(raise_exception=True): + if prediction_ser.is_valid(raise_exception=raise_exception): prediction_ser.save() # add annotations @@ -508,15 +456,8 @@ def add_task(cls, project, maximum_annotations, max_inner_id, storage, link_obje annotation['task'] = task.id annotation['project'] = project.id annotation_ser = AnnotationSerializer(data=annotations, many=True) - - # Always validate annotations, but control error handling based on FF - if annotation_ser.is_valid(): + if annotation_ser.is_valid(raise_exception=raise_exception): annotation_ser.save() - else: - # Log validation errors but don't save invalid annotations - logger.error(f'Invalid annotations for task {task.id}: {annotation_ser.errors}') - if raise_exception: - raise ValidationError(annotation_ser.errors) return task # FIXME: add_annotation_history / post_process_annotations should be here @@ -532,7 +473,6 @@ def _scan_and_create_links(self, link_class): maximum_annotations = self.project.maximum_annotations task = self.project.tasks.order_by('-inner_id').first() max_inner_id = (task.inner_id + 1) if task else 1 - validation_errors = [] # Check feature flag once for the entire sync process check_file_extension = flag_set( @@ -585,28 +525,21 @@ def _scan_and_create_links(self, link_class): for link_object in link_objects: # TODO: batch this loop body with add_task -> add_tasks in a single bulk write. # See DIA-2062 for prerequisites - try: - task = self.add_task( - self.project, - maximum_annotations, - max_inner_id, - self, - link_object, - link_class=link_class, - ) - max_inner_id += 1 + task = self.add_task( + self.project, + maximum_annotations, + max_inner_id, + self, + link_object, + link_class=link_class, + ) + max_inner_id += 1 - # update progress counters for storage info - tasks_created += 1 + # update progress counters for storage info + tasks_created += 1 - # add task to webhook list - tasks_for_webhook.append(task.id) - except ValidationError as e: - # Log validation errors but continue processing other tasks - error_message = f'Validation error for task from {link_object.key}: {e}' - logger.error(error_message) - validation_errors.append(error_message) - continue + # add task to webhook list + tasks_for_webhook.append(task.id) # settings.WEBHOOK_BATCH_SIZE # `WEBHOOK_BATCH_SIZE` sets the maximum number of tasks sent in a single webhook call, ensuring manageable payload sizes. @@ -627,14 +560,9 @@ def _scan_and_create_links(self, link_class): self.project.update_tasks_states( maximum_annotations_changed=False, overlap_cohort_percentage_changed=False, tasks_number_changed=True ) - if validation_errors: - # sync is finished, set completed with errors status for storage info - self.info_set_completed_with_errors( - last_sync_count=tasks_created, tasks_existed=tasks_existed, validation_errors=validation_errors - ) - else: - # sync is finished, set completed status for storage info - self.info_set_completed(last_sync_count=tasks_created, tasks_existed=tasks_existed) + + # sync is finished, set completed status for storage info + self.info_set_completed(last_sync_count=tasks_created, tasks_existed=tasks_existed) def scan_and_create_links(self): """This is proto method - you can override it, or just replace ImportStorageLink by your own model""" diff --git a/label_studio/io_storages/migrations/0020_alter_azureblobexportstorage_status_and_more.py b/label_studio/io_storages/migrations/0020_alter_azureblobexportstorage_status_and_more.py deleted file mode 100644 index efaade0bb5cd..000000000000 --- a/label_studio/io_storages/migrations/0020_alter_azureblobexportstorage_status_and_more.py +++ /dev/null @@ -1,173 +0,0 @@ -# Generated by Django 5.1.10 on 2025-08-04 17:57 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("io_storages", "0019_azureblobimportstoragelink_row_group_and_more"), - ] - - operations = [ - migrations.AlterField( - model_name="azureblobexportstorage", - name="status", - field=models.CharField( - choices=[ - ("initialized", "Initialized"), - ("queued", "Queued"), - ("in_progress", "In progress"), - ("failed", "Failed"), - ("completed", "Completed"), - ("completed_with_errors", "Completed with errors"), - ], - default="initialized", - max_length=64, - ), - ), - migrations.AlterField( - model_name="azureblobimportstorage", - name="status", - field=models.CharField( - choices=[ - ("initialized", "Initialized"), - ("queued", "Queued"), - ("in_progress", "In progress"), - ("failed", "Failed"), - ("completed", "Completed"), - ("completed_with_errors", "Completed with errors"), - ], - default="initialized", - max_length=64, - ), - ), - migrations.AlterField( - model_name="gcsexportstorage", - name="status", - field=models.CharField( - choices=[ - ("initialized", "Initialized"), - ("queued", "Queued"), - ("in_progress", "In progress"), - ("failed", "Failed"), - ("completed", "Completed"), - ("completed_with_errors", "Completed with errors"), - ], - default="initialized", - max_length=64, - ), - ), - migrations.AlterField( - model_name="gcsimportstorage", - name="status", - field=models.CharField( - choices=[ - ("initialized", "Initialized"), - ("queued", "Queued"), - ("in_progress", "In progress"), - ("failed", "Failed"), - ("completed", "Completed"), - ("completed_with_errors", "Completed with errors"), - ], - default="initialized", - max_length=64, - ), - ), - migrations.AlterField( - model_name="localfilesexportstorage", - name="status", - field=models.CharField( - choices=[ - ("initialized", "Initialized"), - ("queued", "Queued"), - ("in_progress", "In progress"), - ("failed", "Failed"), - ("completed", "Completed"), - ("completed_with_errors", "Completed with errors"), - ], - default="initialized", - max_length=64, - ), - ), - migrations.AlterField( - model_name="localfilesimportstorage", - name="status", - field=models.CharField( - choices=[ - ("initialized", "Initialized"), - ("queued", "Queued"), - ("in_progress", "In progress"), - ("failed", "Failed"), - ("completed", "Completed"), - ("completed_with_errors", "Completed with errors"), - ], - default="initialized", - max_length=64, - ), - ), - migrations.AlterField( - model_name="redisexportstorage", - name="status", - field=models.CharField( - choices=[ - ("initialized", "Initialized"), - ("queued", "Queued"), - ("in_progress", "In progress"), - ("failed", "Failed"), - ("completed", "Completed"), - ("completed_with_errors", "Completed with errors"), - ], - default="initialized", - max_length=64, - ), - ), - migrations.AlterField( - model_name="redisimportstorage", - name="status", - field=models.CharField( - choices=[ - ("initialized", "Initialized"), - ("queued", "Queued"), - ("in_progress", "In progress"), - ("failed", "Failed"), - ("completed", "Completed"), - ("completed_with_errors", "Completed with errors"), - ], - default="initialized", - max_length=64, - ), - ), - migrations.AlterField( - model_name="s3exportstorage", - name="status", - field=models.CharField( - choices=[ - ("initialized", "Initialized"), - ("queued", "Queued"), - ("in_progress", "In progress"), - ("failed", "Failed"), - ("completed", "Completed"), - ("completed_with_errors", "Completed with errors"), - ], - default="initialized", - max_length=64, - ), - ), - migrations.AlterField( - model_name="s3importstorage", - name="status", - field=models.CharField( - choices=[ - ("initialized", "Initialized"), - ("queued", "Queued"), - ("in_progress", "In progress"), - ("failed", "Failed"), - ("completed", "Completed"), - ("completed_with_errors", "Completed with errors"), - ], - default="initialized", - max_length=64, - ), - ), - ] diff --git a/label_studio/io_storages/tests/test_multitask_import.py b/label_studio/io_storages/tests/test_multitask_import.py index 99d3756360ff..0df0d73df5ab 100644 --- a/label_studio/io_storages/tests/test_multitask_import.py +++ b/label_studio/io_storages/tests/test_multitask_import.py @@ -206,17 +206,7 @@ def test_storagelink_fields(project, common_task_data): @pytest.fixture def storage(): - project = ProjectFactory( - label_config=""" - - - - - - """ - ) + project = ProjectFactory() storage = S3ImportStorage( project=project, bucket='example', @@ -279,7 +269,7 @@ def create_tasks(storage, params_list: list[StorageObject]): } ], }, - {'data': {'text': 'Prosper annotation helps improve model accuracy.'}}, + {'data': {'text': 'Prosper annotation helps improve model accuracy.'}, 'predictions': [{'result': []}]}, ] diff --git a/label_studio/io_storages/tests/test_storage_prediction_validation.py b/label_studio/io_storages/tests/test_storage_prediction_validation.py deleted file mode 100644 index 59118e712f32..000000000000 --- a/label_studio/io_storages/tests/test_storage_prediction_validation.py +++ /dev/null @@ -1,139 +0,0 @@ -import json - -import boto3 -import pytest -from io_storages.models import S3ImportStorage -from moto import mock_s3 -from projects.tests.factories import ProjectFactory -from rest_framework.test import APIClient - - -@pytest.mark.django_db -class TestStoragePredictionValidation: - """Test prediction validation in cloud storage imports.""" - - @pytest.fixture - def project(self): - """Create a project with a label config for prediction validation.""" - return ProjectFactory( - label_config=""" - - - - - - - - """ - ) - - @pytest.fixture - def api_client(self): - """Create API client for testing.""" - return APIClient() - - def test_storage_import_with_valid_prediction(self, project, api_client): - """Test that storage import accepts valid predictions.""" - # Setup API client - api_client.force_authenticate(user=project.created_by) - - # Create valid task data with prediction - valid_task_data = { - 'data': {'text': 'This is a positive review'}, - 'predictions': [ - { - 'result': [ - { - 'from_name': 'sentiment', - 'to_name': 'text', - 'type': 'choices', - 'value': {'choices': ['positive']}, - } - ], - 'score': 0.95, - 'model_version': 'v1.0', - } - ], - } - - with mock_s3(): - # Setup S3 bucket and test data - s3 = boto3.client('s3', region_name='us-east-1') - bucket_name = 'pytest-s3-prediction-validation' - s3.create_bucket(Bucket=bucket_name) - - # Put valid test data into S3 - s3.put_object(Bucket=bucket_name, Key='valid_prediction.json', Body=json.dumps([valid_task_data])) - - # Create storage and sync - storage = S3ImportStorage( - project=project, - bucket=bucket_name, - aws_access_key_id='example', - aws_secret_access_key='example', - use_blob_urls=False, - ) - storage.save() - storage.sync() - - # Verify task was created - tasks_response = api_client.get(f'/api/tasks?project={project.id}') - assert tasks_response.status_code == 200 - tasks = tasks_response.json()['tasks'] - assert len(tasks) == 1 - - # Verify prediction was created - predictions_response = api_client.get(f'/api/predictions?task={tasks[0]["id"]}') - assert predictions_response.status_code == 200 - predictions = predictions_response.json() - assert len(predictions) == 1 - - def test_storage_import_with_invalid_prediction(self, project, api_client): - """Test that storage import rejects invalid predictions.""" - # Setup API client - api_client.force_authenticate(user=project.created_by) - - # Create invalid task data with prediction (wrong from_name) - invalid_task_data = { - 'data': {'text': 'This is a positive review'}, - 'predictions': [ - { - 'result': [ - { - 'from_name': 'nonexistent_tag', # Invalid from_name - 'to_name': 'text', - 'type': 'choices', - 'value': {'choices': ['positive']}, - } - ], - 'score': 0.95, - 'model_version': 'v1.0', - } - ], - } - - with mock_s3(): - # Setup S3 bucket and test data - s3 = boto3.client('s3', region_name='us-east-1') - bucket_name = 'pytest-s3-prediction-validation' - s3.create_bucket(Bucket=bucket_name) - - # Put invalid test data into S3 - s3.put_object(Bucket=bucket_name, Key='invalid_prediction.json', Body=json.dumps([invalid_task_data])) - - # Create storage and sync - storage = S3ImportStorage( - project=project, - bucket=bucket_name, - aws_access_key_id='example', - aws_secret_access_key='example', - use_blob_urls=False, - ) - storage.save() - storage.sync() - - # Verify task was NOT created due to validation failure - tasks_response = api_client.get(f'/api/tasks?project={project.id}') - assert tasks_response.status_code == 200 - tasks = tasks_response.json()['tasks'] - assert len(tasks) == 0 # No tasks should be created when predictions are invalid diff --git a/label_studio/tasks/serializers.py b/label_studio/tasks/serializers.py index fa006fbbcd63..b40652a28a08 100644 --- a/label_studio/tasks/serializers.py +++ b/label_studio/tasks/serializers.py @@ -10,7 +10,6 @@ from django.conf import settings from django.db import IntegrityError, transaction from drf_spectacular.utils import extend_schema_field -from label_studio_sdk.label_interface import LabelInterface from projects.models import Project from rest_flex_fields import FlexFieldsModelSerializer from rest_framework import generics, serializers @@ -74,31 +73,6 @@ class PredictionSerializer(ModelSerializer): ) created_ago = serializers.CharField(default='', read_only=True, help_text='Delta time from creation time') - def validate(self, data): - """Validate prediction using LabelInterface against project configuration""" - # Only validate if we're updating the result field - if 'result' not in data: - return data - - # Get the project from the task or directly from data - project = None - if 'task' in data: - project = data['task'].project - elif 'project' in data: - project = data['project'] - - if not project: - raise ValidationError('Project is required for prediction validation') - - # Validate prediction using LabelInterface - li = LabelInterface(project.label_config) - validation_errors = li.validate_prediction(data, return_errors=True) - - if validation_errors: - raise ValidationError(f'Error validating prediction: {validation_errors}') - - return data - class Meta: model = Prediction fields = '__all__' @@ -421,11 +395,7 @@ def create(self, validated_data): db_tasks = self.add_tasks(task_annotations, task_predictions, validated_tasks) db_annotations = self.add_annotations(task_annotations, user) - prediction_errors = self.add_predictions(task_predictions) - - # If there are prediction validation errors, raise them - if prediction_errors: - raise ValidationError({'predictions': prediction_errors}) + self.add_predictions(task_predictions) self.post_process_annotations(user, db_annotations, 'imported') self.post_process_tasks(self.project.id, [t.id for t in self.db_tasks]) @@ -446,64 +416,36 @@ def create(self, validated_data): def add_predictions(self, task_predictions): """Save predictions to DB and set the latest model version in the project""" db_predictions = [] - validation_errors = [] - - should_validate = self.project.label_config_is_not_default # add predictions last_model_version = None for i, predictions in enumerate(task_predictions): - for j, prediction in enumerate(predictions): + for prediction in predictions: if not isinstance(prediction, dict): - validation_errors.append(f'Task {i}, prediction {j}: Prediction must be a dictionary') continue - # Validate prediction only when project label config is not default - if should_validate: + # we need to call result normalizer here since "bulk_create" doesn't call save() method + result = Prediction.prepare_prediction_result(prediction['result'], self.project) + prediction_score = prediction.get('score') + if prediction_score is not None: try: - li = LabelInterface(self.project.label_config) if should_validate else None - validation_errors_list = li.validate_prediction(prediction, return_errors=True) - - if validation_errors_list: - # Format errors for better readability - for error in validation_errors_list: - validation_errors.append(f'Task {i}, prediction {j}: {error}') - continue - - except Exception as e: - validation_errors.append(f'Task {i}, prediction {j}: Error validating prediction - {str(e)}') - continue - - try: - # we need to call result normalizer here since "bulk_create" doesn't call save() method - result = Prediction.prepare_prediction_result(prediction['result'], self.project) - prediction_score = prediction.get('score') - if prediction_score is not None: - try: - prediction_score = float(prediction_score) - except ValueError: - logger.error( - "Can't upload prediction score: should be in float format." 'Fallback to score=None' - ) - prediction_score = None - - last_model_version = prediction.get('model_version', 'undefined') - db_predictions.append( - Prediction( - task=self.db_tasks[i], - project=self.db_tasks[i].project, - result=result, - score=prediction_score, - model_version=last_model_version, + prediction_score = float(prediction_score) + except ValueError: + logger.error( + "Can't upload prediction score: should be in float format." 'Fallback to score=None' ) + prediction_score = None + + last_model_version = prediction.get('model_version', 'undefined') + db_predictions.append( + Prediction( + task=self.db_tasks[i], + project=self.db_tasks[i].project, + result=result, + score=prediction_score, + model_version=last_model_version, ) - except Exception as e: - validation_errors.append(f'Task {i}, prediction {j}: Failed to create prediction - {str(e)}') - continue - - # Return validation errors if they exist - if validation_errors: - return validation_errors + ) # predictions: DB bulk create self.db_predictions = Prediction.objects.bulk_create(db_predictions, batch_size=settings.BATCH_SIZE) @@ -514,7 +456,7 @@ def add_predictions(self, task_predictions): self.project.model_version = last_model_version self.project.save() - return None # No errors + return self.db_predictions, last_model_version def add_reviews(self, task_reviews, annotation_mapping, project): """Save task reviews to DB""" diff --git a/label_studio/tests/conftest.py b/label_studio/tests/conftest.py index 14b3ddccb187..c51565046976 100644 --- a/label_studio/tests/conftest.py +++ b/label_studio/tests/conftest.py @@ -344,7 +344,7 @@ def ml_backend_for_test_predict(ml_backend): 'model_version': 'ModelSingle', 'score': 0.1, 'result': [ - {'from_name': 'label', 'to_name': 'text', 'type': 'choices', 'value': {'choices': ['label_A']}} + {'from_name': 'label', 'to_name': 'text', 'type': 'choices', 'value': {'choices': ['Single']}} ], }, ] @@ -445,7 +445,7 @@ def project_dialog():
- +
""" diff --git a/label_studio/tests/data_import.tavern.yml b/label_studio/tests/data_import.tavern.yml index 0cba5d141da8..7512843e3a54 100644 --- a/label_studio/tests/data_import.tavern.yml +++ b/label_studio/tests/data_import.tavern.yml @@ -120,7 +120,7 @@ stages: - name: stage request: data: - label_config: + label_config: title: Check consistency of imported data columns method: POST url: '{django_live_url}/api/projects' @@ -535,7 +535,7 @@ stages: - name: Create project request: data: - label_config: title: Image Classification Project method: POST @@ -571,7 +571,7 @@ stages: drafts: [] predictions: - result: - - from_name: bbox + - from_name: objects to_name: image type: rectanglelabels value: @@ -627,22 +627,10 @@ stages: url: '{django_live_url}/api/projects/{project_pk}/import/predictions' json: - task: !int '{first_task}' - result: - - from_name: label - to_name: image - type: choices - value: - choices: - - AAA + result: AAA score: 0.123 - task: !int '{third_task}' - result: - - from_name: label - to_name: image - type: choices - value: - choices: - - BBB + result: BBB model_version: abcdefgh response: status_code: 201 diff --git a/label_studio/tests/data_manager/api_tasks.tavern.yml b/label_studio/tests/data_manager/api_tasks.tavern.yml index f45fe546cfe5..b004896d15cf 100644 --- a/label_studio/tests/data_manager/api_tasks.tavern.yml +++ b/label_studio/tests/data_manager/api_tasks.tavern.yml @@ -16,7 +16,6 @@ stages: data: title: Test Draft 1 show_collab_predictions: true - label_config: '' method: POST url: '{django_live_url}/api/projects' response: @@ -356,7 +355,6 @@ stages: data: title: Test Draft 1 show_collab_predictions: true - label_config: '' method: POST url: '{django_live_url}/api/projects' response: @@ -711,7 +709,6 @@ stages: data: title: Test Draft 1 show_collab_predictions: true - label_config: '' method: POST url: '{django_live_url}/api/projects' response: @@ -1141,7 +1138,6 @@ stages: data: title: Test Draft 1 show_collab_predictions: true - label_config: '' method: POST url: '{django_live_url}/api/projects' response: @@ -1506,7 +1502,6 @@ stages: data: title: Test Draft 1 show_collab_predictions: true - label_config: '' method: POST url: '{django_live_url}/api/projects' response: @@ -1625,7 +1620,6 @@ stages: data: title: Test Draft 1 show_collab_predictions: true - label_config: '' method: POST url: '{django_live_url}/api/projects' response: @@ -1723,7 +1717,6 @@ stages: data: title: Test Draft 1 show_collab_predictions: true - label_config: '' method: POST url: '{django_live_url}/api/projects' response: @@ -1915,7 +1908,6 @@ stages: data: title: Test Draft 1 show_collab_predictions: true - label_config: '' method: POST url: '{django_live_url}/api/projects' response: diff --git a/label_studio/tests/data_manager/filters/int_tasks.json b/label_studio/tests/data_manager/filters/int_tasks.json index 46e90c21c5a5..4f641a961437 100644 --- a/label_studio/tests/data_manager/filters/int_tasks.json +++ b/label_studio/tests/data_manager/filters/int_tasks.json @@ -8,12 +8,12 @@ { "result": [ { - "from_name": "label", + "from_name": "text_class", "to_name": "text", "type": "choices", "value": { "choices": [ - "pos" + "class_A" ] } } diff --git a/label_studio/tests/data_manager/test_ordering_filters.py b/label_studio/tests/data_manager/test_ordering_filters.py index ff6c91f34f6d..62e2b3968941 100644 --- a/label_studio/tests/data_manager/test_ordering_filters.py +++ b/label_studio/tests/data_manager/test_ordering_filters.py @@ -66,32 +66,18 @@ def test_views_ordering(ordering, element_index, undefined, business_client, pro user=project.created_by, project=project, file=ContentFile('', name='file_upload1') ) - task_id_1 = make_task({'data': {task_field_name: 1, 'data': 1}, 'file_upload': file_upload1}, project).id + task_id_1 = make_task({'data': {task_field_name: 1}, 'file_upload': file_upload1}, project).id make_annotation({'result': [{'1': True}]}, task_id_1) - make_prediction( - { - 'result': [{'from_name': 'test_batch_predictions', 'to_name': 'text', 'value': {'choices': ['class_A']}}], - 'score': 0.5, - }, - task_id_1, - ) + make_prediction({'result': [{'1': True}], 'score': 1}, task_id_1) file_upload2 = FileUpload.objects.create( user=project.created_by, project=project, file=ContentFile('', name='file_upload2') ) - task_id_2 = make_task({'data': {task_field_name: 2, 'data': 2}, 'file_upload': file_upload2}, project).id + task_id_2 = make_task({'data': {task_field_name: 2}, 'file_upload': file_upload2}, project).id for _ in range(0, 2): make_annotation({'result': [{'2': True}], 'was_cancelled': True}, task_id_2) for _ in range(0, 2): - make_prediction( - { - 'result': [ - {'from_name': 'test_batch_predictions', 'to_name': 'text', 'value': {'choices': ['class_B']}} - ], - 'score': 1, - }, - task_id_2, - ) + make_prediction({'result': [{'2': True}], 'score': 2}, task_id_2) task_ids = [task_id_1, task_id_2] @@ -350,61 +336,29 @@ def test_views_filters(filters, ids, business_client, project_id): task_data_field_name = settings.DATA_UNDEFINED_NAME - task_id_1 = make_task({'data': {task_data_field_name: 'some text1', 'data': 'some text1'}}, project).id + task_id_1 = make_task({'data': {task_data_field_name: 'some text1'}}, project).id make_annotation( - { - 'result': [ - { - 'from_name': 'test_batch_predictions', - 'to_name': 'text', - 'value': {'choices': ['class_A']}, - 'text': 'first annotation', - } - ], - 'completed_by': ann1, - }, - task_id_1, - ) - make_prediction( - { - 'result': [{'from_name': 'test_batch_predictions', 'to_name': 'text', 'value': {'choices': ['class_A']}}], - 'score': 1, - }, - task_id_1, + {'result': [{'from_name': '1_first', 'to_name': '', 'value': {}}], 'completed_by': ann1}, task_id_1 ) + make_prediction({'result': [{'from_name': '1_first', 'to_name': '', 'value': {}}], 'score': 1}, task_id_1) - task_id_2 = make_task({'data': {task_data_field_name: 'some text2', 'data': 'some text2'}}, project).id + task_id_2 = make_task({'data': {task_data_field_name: 'some text2'}}, project).id for ann in (ann1, ann2): make_annotation( { - 'result': [ - { - 'from_name': 'test_batch_predictions', - 'to_name': 'text', - 'value': {'choices': ['class_B']}, - 'text': 'second annotation', - } - ], + 'result': [{'from_name': '2_second', 'to_name': '', 'value': {}}], 'was_cancelled': True, 'completed_by': ann, }, task_id_2, ) for _ in range(0, 2): - make_prediction( - { - 'result': [ - {'from_name': 'test_batch_predictions', 'to_name': 'text', 'value': {'choices': ['class_B']}} - ], - 'score': 2, - }, - task_id_2, - ) + make_prediction({'result': [{'from_name': '2_second', 'to_name': '', 'value': {}}], 'score': 2}, task_id_2) task_ids = [0, task_id_1, task_id_2] for _ in range(0, 2): - task_id = make_task({'data': {task_data_field_name: 'some text_', 'data': 'some text_'}}, project).id + task_id = make_task({'data': {task_data_field_name: 'some text_'}}, project).id task_ids.append(task_id) for item in filters['items']: diff --git a/label_studio/tests/data_manager/test_undefined.py b/label_studio/tests/data_manager/test_undefined.py index f1a7c21201c8..cc59be972b69 100644 --- a/label_studio/tests/data_manager/test_undefined.py +++ b/label_studio/tests/data_manager/test_undefined.py @@ -54,8 +54,6 @@ def test_views_filters_with_undefined(business_client, project_id): 14. Filter by "image" with "photo" should return task 1 and task 2 """ project = Project.objects.get(pk=project_id) - project.label_config = '' - project.save() # Step 1: Import task 1: {"$undefined$": "photo1.jpg"} task_data_field_name = settings.DATA_UNDEFINED_NAME # "$undefined$" diff --git a/label_studio/tests/ml/test_predict.py b/label_studio/tests/ml/test_predict.py index 997e2dc698db..3107139e9688 100644 --- a/label_studio/tests/ml/test_predict.py +++ b/label_studio/tests/ml/test_predict.py @@ -13,7 +13,7 @@ def test_get_single_prediction_on_task(business_client, ml_backend_for_test_pred label_config=""" - + @@ -43,7 +43,7 @@ def test_get_single_prediction_on_task(business_client, ml_backend_for_test_pred # ensure task has a single prediction with the correct value assert len(payload['predictions']) == 1 - assert payload['predictions'][0]['result'][0]['value']['choices'][0] == 'label_A' + assert payload['predictions'][0]['result'][0]['value']['choices'][0] == 'Single' assert payload['predictions'][0]['model_version'] == 'ModelSingle' @@ -55,7 +55,7 @@ def test_get_multiple_predictions_on_task(business_client, ml_backend_for_test_p label_config=""" - + diff --git a/label_studio/tests/next_task.tavern.yml b/label_studio/tests/next_task.tavern.yml index 6acba758d506..db2967ad03fd 100644 --- a/label_studio/tests/next_task.tavern.yml +++ b/label_studio/tests/next_task.tavern.yml @@ -44,12 +44,12 @@ stages: text: Test example phrase predictions: # last taken model version is activated - result: - - from_name: label + - from_name: text_class to_name: text type: choices value: choices: - - neg + - class_B model_version: model_version_B - name: change_project_model_version @@ -72,12 +72,12 @@ stages: is_labeled: false predictions: - result: - - from_name: label + - from_name: text_class to_name: text type: choices value: choices: - - pos + - class_A model_version: model_version_A - name: change_project_model_version diff --git a/label_studio/tests/predictions.model.tavern.yml b/label_studio/tests/predictions.model.tavern.yml index f8e35bf0969f..6545721caa8f 100644 --- a/label_studio/tests/predictions.model.tavern.yml +++ b/label_studio/tests/predictions.model.tavern.yml @@ -26,9 +26,9 @@ stages: result: - value: choices: - - neg + - Negative id: qpQHs3Yy4K - from_name: label + from_name: sentiment to_name: text type: choices model_version: 1 @@ -52,9 +52,9 @@ stages: result: - value: choices: - - neg + - Negative id: qpQHs3Yy4K - from_name: label + from_name: sentiment to_name: text type: choices model_version: 1 @@ -78,9 +78,9 @@ stages: result: - value: choices: - - neg + - Negative id: qpQHs3Yy4K - from_name: label + from_name: sentiment to_name: text type: choices status_code: 200 @@ -177,9 +177,9 @@ stages: result: - value: choices: - - neg + - Negative id: qpQHs3Yy4K - from_name: label + from_name: sentiment to_name: text type: choices model_version: 1 @@ -203,9 +203,9 @@ stages: result: - value: choices: - - neg + - Negative id: qpQHs3Yy4K - from_name: label + from_name: sentiment to_name: text type: choices model_version: 1 @@ -229,9 +229,9 @@ stages: result: - value: choices: - - neg + - Negative id: qpQHs3Yy4K - from_name: label + from_name: sentiment to_name: text type: choices status_code: 200 diff --git a/label_studio/tests/predictions.tavern.yml b/label_studio/tests/predictions.tavern.yml index d68aee838478..fb3901db162a 100644 --- a/label_studio/tests/predictions.tavern.yml +++ b/label_studio/tests/predictions.tavern.yml @@ -18,9 +18,9 @@ stages: result: - value: choices: - - neg + - Negative id: qpQHs3Yy4K - from_name: label + from_name: sentiment to_name: text type: choices model_version: 'test' @@ -43,9 +43,9 @@ stages: result: - value: choices: - - neg + - Negative id: qpQHs3Yy4K - from_name: label + from_name: sentiment to_name: text type: choices status_code: 200 @@ -61,9 +61,9 @@ stages: result: - value: choices: - - neg + - Negative id: qpQHs3Yy4K - from_name: label + from_name: sentiment to_name: text type: choices status_code: 200 @@ -129,9 +129,9 @@ stages: result: - value: choices: - - neg + - Negative id: qpQHs3Yy4K - from_name: label + from_name: sentiment to_name: text type: choices model_version: "test" @@ -179,14 +179,7 @@ stages: name: create_prediction request: json: - result: - - value: - choices: - - neg - id: qpQHs3Yy4K - from_name: label - to_name: text - type: choices + result: Negative score: 0.987 model_version: test_model task: '{task_pk}' @@ -208,7 +201,7 @@ stages: result: - value: choices: - - neg + - Negative from_name: label to_name: text type: choices diff --git a/label_studio/tests/sdk/test_ml.py b/label_studio/tests/sdk/test_ml.py index d0f3e791a7e6..224994c4cbd8 100644 --- a/label_studio/tests/sdk/test_ml.py +++ b/label_studio/tests/sdk/test_ml.py @@ -52,11 +52,11 @@ def test_batch_predictions_single_prediction_per_task(django_live_url, business_ assert len(predictions) == 2 # check that the first prediction has the correct value - assert predictions[0].result[0]['value']['choices'][0] == 'label_A' + assert predictions[0].result[0]['value']['choices'][0] == 'Single' assert predictions[0].model_version == 'ModelSingle' # check that the second prediction has the correct value - assert predictions[1].result[0]['value']['choices'][0] == 'label_A' + assert predictions[1].result[0]['value']['choices'][0] == 'Single' assert predictions[1].model_version == 'ModelSingle' # additionally let's test actions: convert predictions to annotations @@ -81,10 +81,10 @@ def test_batch_predictions_single_prediction_per_task(django_live_url, business_ assert not task.predictions else: assert len(task.annotations) == 1 - assert task.annotations[0]['result'][0]['value']['choices'][0] == 'label_A' + assert task.annotations[0]['result'][0]['value']['choices'][0] == 'Single' assert len(task.predictions) == 1 - assert task.predictions[0].result[0]['value']['choices'][0] == 'label_A' + assert task.predictions[0].result[0]['value']['choices'][0] == 'Single' assert task.predictions[0].model_version == 'ModelSingle' assert task.predictions[0].score == 0.1 diff --git a/label_studio/tests/sdk/test_prediction_validation.py b/label_studio/tests/sdk/test_prediction_validation.py deleted file mode 100644 index 68ce72a8151c..000000000000 --- a/label_studio/tests/sdk/test_prediction_validation.py +++ /dev/null @@ -1,522 +0,0 @@ -import pytest -from django.contrib.auth import get_user_model -from label_studio_sdk import LabelStudio -from organizations.tests.factories import OrganizationFactory -from projects.tests.factories import ProjectFactory -from tasks.tests.factories import TaskFactory -from users.tests.factories import UserFactory - -User = get_user_model() - - -class TestSDKPredictionValidation: - """Comprehensive tests for prediction validation using Label Studio SDK""" - - @pytest.fixture(autouse=True) - def setup(self, django_db_setup, django_db_blocker): - """Set up test environment with user, organization, project, and task using factories""" - with django_db_blocker.unblock(): - self.user = UserFactory() - self.organization = OrganizationFactory(created_by=self.user) - self.user.active_organization = self.organization - self.user.save() - - # Create a project with a comprehensive label configuration - self.project = ProjectFactory( - title='Test Project', - label_config=""" - - - - - - - - - - - - - - - - -