Increment major version automatically

Author: Deepthi-Chand

.pre-commit-config.yaml

@@ -34,6 +34,8 @@ repos:
           - django-stubs
           - djangorestframework-stubs
           - pandas-stubs
+          - pandas
+          - deepdiff
           - types-redis
           - strawberry-graphql
           - strawberry-graphql-django

api/models/Resource.py

@@ -1,6 +1,7 @@
 import os
 import random
 import uuid
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 
 import structlog
@@ -15,6 +16,7 @@
 
 from api.managers.dvc_manager import DVCManager
 from api.utils.enums import DataType
+from api.utils.version_detection import detect_version_change_type
 
 if TYPE_CHECKING:
     from api.models.Dataset import Dataset
@@ -162,7 +164,7 @@ def version_resource_with_dvc(sender, instance: ResourceFileDetails, created, **
                 )
                 return
 
-            # Determine version number using semantic versioning
+            # Get the latest version
             last_version: Optional[ResourceVersion] = (
                 instance.resource.versions.order_by("-created_at").first()
             )
@@ -171,9 +173,64 @@ def version_resource_with_dvc(sender, instance: ResourceFileDetails, created, **
             if last_version is None:
                 new_version = "v1.0.0"
             else:
-                # Default to minor version increment, could be configurable in the future
+                # Determine appropriate version increment type by analyzing changes
+                try:
+                    # Get the previous version file path
+                    # We need to create a temporary copy of the previous version
+                    import shutil
+                    import tempfile
+
+                    from django.core.files.storage import default_storage
+
+                    # Create a temporary directory for the previous version
+                    with tempfile.TemporaryDirectory() as temp_dir:
+                        # Get the previous version file path
+                        prev_file_name = f"prev_version_{instance.resource.id}.{instance.file.name.split('.')[-1]}"
+                        prev_file_path = os.path.join(temp_dir, prev_file_name)
+
+                        # Use DVC to get the previous version
+                        try:
+                            # Try to checkout the previous version using DVC
+                            rel_path = Path(instance.file.path).relative_to(
+                                settings.DVC_REPO_PATH
+                            )
+                            tag_name = f"{instance.resource.name}-{last_version.version_number}"
+
+                            # Save current file to temp location
+                            current_file_path = instance.file.path
+                            temp_current = os.path.join(temp_dir, "current_file")
+                            shutil.copy2(current_file_path, temp_current)
+
+                            # Checkout previous version
+                            dvc.rollback_to_version(instance.file.path, tag_name)
+
+                            # Copy the previous version to our temp location
+                            shutil.copy2(instance.file.path, prev_file_path)
+
+                            # Restore current file
+                            shutil.copy2(temp_current, current_file_path)
+
+                            # Detect version change type
+                            increment_type = detect_version_change_type(
+                                prev_file_path, current_file_path
+                            )
+                            logger.info(
+                                f"Detected version change type: {increment_type} for {instance.resource.name}"
+                            )
+                        except Exception as e:
+                            logger.warning(
+                                f"Could not analyze version changes: {str(e)}, defaulting to minor version"
+                            )
+                            increment_type = "minor"
+                except Exception as e:
+                    logger.warning(
+                        f"Error in version detection: {str(e)}, defaulting to minor version"
+                    )
+                    increment_type = "minor"
+
+                # Increment version based on detected change type
                 new_version = _increment_version(
-                    last_version.version_number, increment_type="minor"
+                    last_version.version_number, increment_type=increment_type
                 )
 
             # Use chunked mode for large files (over 100MB)

api/models/__init__.py

@@ -10,6 +10,7 @@
     ResourceDataTable,
     ResourceFileDetails,
     ResourcePreviewDetails,
+    ResourceVersion,
 )
 from api.models.ResourceChartDetails import ResourceChartDetails
 from api.models.ResourceChartImage import ResourceChartImage

api/schema/resource_schema.py

@@ -12,13 +12,16 @@
 from strawberry.file_uploads import Upload
 from strawberry.types import Info
 
+from api.managers.dvc_manager import DVCManager
 from api.models import (
     Dataset,
     Resource,
     ResourceFileDetails,
     ResourcePreviewDetails,
     ResourceSchema,
+    ResourceVersion,
 )
+from api.models.Resource import _increment_version
 from api.types.type_resource import TypeResource
 from api.utils.constants import FORMAT_MAPPING
 from api.utils.data_indexing import index_resource_data
@@ -64,6 +67,16 @@ class UpdateFileResourceInput:
     preview_details: Optional[PreviewDetails] = strawberry.field(default=None)
 
 
+@strawberry.input
+class CreateMajorVersionInput:
+    """Input type for creating a major version of a resource."""
+
+    resource_id: uuid.UUID = strawberry.field()
+    description: str = strawberry.field(
+        description="Description of the changes in this major version"
+    )
+
+
 @strawberry.enum
 class FieldType(Enum):
     """Enum for field types."""
@@ -339,3 +352,71 @@ def delete_file_resource(self, info: Info, resource_id: uuid.UUID) -> bool:
             return True
         except Resource.DoesNotExist as e:
             raise ValueError(f"Resource with ID {resource_id} does not exist.")
+
+    @strawberry_django.mutation(handle_django_errors=True)
+    @trace_resolver(name="create_major_version", attributes={"component": "resource"})
+    def create_major_version(
+        self, info: Info, input: CreateMajorVersionInput
+    ) -> TypeResource:
+        """Create a major version for a resource.
+
+        This should be used when significant changes are made to the resource data structure,
+        such as schema changes, column additions/removals, or other breaking changes.
+        """
+        import os
+
+        from django.conf import settings
+
+        try:
+            # Get the resource
+            resource = Resource.objects.get(id=input.resource_id)
+        except Resource.DoesNotExist:
+            raise ValueError(f"Resource with ID {input.resource_id} does not exist")
+
+        # Get the latest version
+        last_version = resource.versions.order_by("-created_at").first()
+
+        if not last_version:
+            logger.warning(
+                f"No previous version found for resource {resource.name}, creating initial version"
+            )
+            new_version = "v1.0.0"
+        else:
+            # Increment major version
+            new_version = _increment_version(
+                last_version.version_number, increment_type="major"
+            )
+
+        # Initialize DVC manager
+        dvc = DVCManager(settings.DVC_REPO_PATH)
+
+        # Get the resource file path
+        file_path = resource.resourcefiledetails.file.path
+
+        # Determine if file is large and should use chunking
+        file_size = (
+            resource.resourcefiledetails.file.size
+            if hasattr(resource.resourcefiledetails.file, "size")
+            else os.path.getsize(file_path)
+        )
+        use_chunked = file_size > 100 * 1024 * 1024  # 100MB threshold
+
+        # Track with DVC
+        dvc_file = dvc.track_resource(file_path, chunked=use_chunked)
+        message = f"Major version update for resource: {resource.name} to version {new_version}"
+        dvc.commit_version(dvc_file, message)
+        dvc.tag_version(f"{resource.name}-{new_version}")
+
+        # Create version record
+        ResourceVersion.objects.create(
+            resource=resource,
+            version_number=new_version,
+            change_description=input.description,
+        )
+
+        # Update resource version field
+        resource.version = new_version
+        resource.save(update_fields=["version"])
+
+        logger.info(f"Created major version {new_version} for resource {resource.name}")
+        return TypeResource.from_django(resource)

api/types/charts/base_chart.py

@@ -623,7 +623,7 @@ def _get_y_values(
                         if pd.notna(value):
                             x_to_y_map[x_val] = float(value)
                 elif pd.notna(y_val):
-                    x_to_y_map[x_val] = float(y_val)
+                    x_to_y_map[x_val] = float(y_val)  # type: ignore
             except (ValueError, TypeError) as e:
                 logger.warning(f"Error converting y-value for {x_val}: {e}")
             except Exception as e:

api/types/type_usecase.py

@@ -13,7 +13,7 @@
 from api.types.type_usecase_metadata import TypeUseCaseMetadata
 from api.utils.enums import UseCaseStatus
 
-use_case_status: EnumType = strawberry.enum(UseCaseStatus)
+use_case_status = strawberry.enum(UseCaseStatus)  # type: ignore
 
 
 @strawberry_django.filter(UseCase)

api/urls.py

@@ -31,11 +31,11 @@
             )
         ),
     ),
-    re_path(
+    re_path(  # type: ignore
         r"download/(?P<type>resource|access_resource|chart|chart_image)/(?P<id>[0-9a-f]{8}\-[0-9a-f]{4}\-4[0-9a-f]{3}\-[89ab][0-9a-f]{3}\-[0-9a-f]{12})",
         download,
     ),
-    re_path(
+    re_path(  # type: ignore
         r"generate-dynamic-chart/(?P<resource_id>[0-9a-f]{8}\-[0-9a-f]{4}\-4[0-9a-f]{3}\-[89ab][0-9a-f]{3}\-[0-9a-f]{12})",
         generate_dynamic_chart,
         name="generate_dynamic_chart",

api/utils/elasticsearch_telemetry_patch.py

@@ -47,4 +47,4 @@ def patch_elasticsearch_instrumentation() -> None:
     from opentelemetry.instrumentation.elasticsearch import utils
 
     # Replace the internal _flatten_dict function with our patched version
-    utils._flatten_dict = patched_flatten_dict
+    utils._flatten_dict = patched_flatten_dict  # type: ignore