feat(LAB-4125): in consensus labeling step, I export all annotators' latest labeled versions

Author: FannyGaudin

pyproject.toml

@@ -44,7 +44,7 @@ dependencies = [
   "filelock >= 3.0.0, < 4.0.0",
   "pip-system-certs >= 4.0.0, < 5.0.0; platform_system=='Windows'",
   "pyrate-limiter >= 3, < 4",
-  "kili-formats == 1.3.0"
+  "kili-formats == 1.4.0"
 ]
 urls = { homepage = "https://github.com/kili-technology/kili-python-sdk" }
 
@@ -63,7 +63,7 @@ dev = [
   "pytest-xdist[psutil]",
   "pytest-timeout",
   # documentation
-  "mkdocs",
+  "mkdocs >= 1.0.0, < 2.0.0",
   "mkdocs-material",
   "mkdocstrings",
   "mkdocstrings-python-legacy",
@@ -79,7 +79,7 @@ dev = [
   "nbconvert",
   "ipykernel",
   # optional dependencies
-  "kili-formats[all] == 1.3.0",
+  "kili-formats[all] == 1.4.0",
   "opencv-python >= 4.0.0, < 5.0.0",
   "azure-storage-blob >= 12.0.0, < 13.0.0",
   # optional dependencies gis
@@ -90,14 +90,14 @@ dev = [
   # optional dependencies - yolo
   "pyyaml >= 6.0, < 7.0",
   # optional dependencies - image
-  "Pillow >=9.0.0, <11.0.0"
+  "Pillow >=10.0.0, <13.0.0"
 ]
 all = [
   # aggregate all optional deps without dev
   "azure-storage-blob >= 12.0.0, < 13.0.0",
-  "kili-formats[all] == 1.3.0",
+  "kili-formats[all] == 1.4.0",
   "opencv-python >= 4.0.0, < 5.0.0",
-  "Pillow >=9.0.0, <11.0.0",
+  "Pillow >=10.0.0, <13.0.0",
   "pyproj == 3.7.1",
   "pyyaml >= 6.0, < 7.0",
   "shapely >= 1.8, < 3",
@@ -108,20 +108,20 @@ cli = [
   "tabulate >= 0.9.0, < 0.10.0"
 ]
 coco = [
-  "kili-formats[coco] == 1.3.0"
+  "kili-formats[coco] == 1.4.0"
 ]
 gis = [
   "pyproj == 3.7.1",
   "shapely >= 1.8, < 3"
 ]
 image = [
-  "Pillow >=9.0.0, <11.0.0",
-  "kili-formats[image] == 1.3.0"
+  "Pillow >=10.0.0, <13.0.0",
+  "kili-formats[image] == 1.4.0"
 ]
 image-utils = ["opencv-python >= 4.0.0, < 5.0.0"]
 
 video = [
-  "kili-formats[video] == 1.3.0"
+  "kili-formats[video] == 1.4.0"
 ]
 yolo = [
   "pyyaml >= 6.0, < 7.0"

src/kili/adapters/kili_api_gateway/asset/formatters.py

@@ -139,6 +139,11 @@ def load_asset_json_fields(asset: dict, fields: ListOrTuple[str], http_client: H
     if "latestLabel.jsonResponse" in fields and asset.get("latestLabel") is not None:
         _process_label_json_response(asset["latestLabel"], url_to_label_mapping)
 
+    if "latestLabels.jsonResponse" in fields:
+        for label in asset.get("latestLabels", []):
+            if label is not None:
+                _process_label_json_response(label, url_to_label_mapping)
+
     if url_to_label_mapping:
         download_json_responses_parallel(url_to_label_mapping, http_client)
 

src/kili/domain_api/exports.py

@@ -12,7 +12,7 @@
 from kili.domain.types import ListOrTuple
 from kili.domain_api.base import DomainNamespace
 from kili.domain_api.namespace_utils import get_available_methods
-from kili.services.export.types import CocoAnnotationModifier, LabelFormat, SplitOption
+from kili.services.export.types import CocoAnnotationModifier, ExportType, LabelFormat, SplitOption
 
 if TYPE_CHECKING:
     import pandas as pd
@@ -94,6 +94,7 @@ def kili(
         include_sent_back_labels: Optional[bool] = None,
         label_type_in: Optional[list[LabelType]] = None,
         single_file: Optional[bool] = False,
+        export_type: ExportType = "latest_from_last_step",
     ):
         """Export project labels in Kili native format.
 
@@ -114,6 +115,14 @@ def kili(
                 whose type belongs to that list.
                 By default, only `DEFAULT` and `REVIEW` labels are exported.
             single_file: If True, all labels are exported in a single JSON file.
+            export_type: Type of export. Options are:
+                - `"latest_from_last_step"`: exports the latest label from the last workflow step
+                  (uses `latestLabels` field, supports multiple annotators).
+                - `"latest_from_all_steps"`: exports latest labels from all workflow steps
+                  (useful for consensus projects with multiple annotators).
+                - `"latest"`: exports the single latest label per asset (uses `latestLabel` field).
+                - `"normal"`: exports all labels for each asset.
+                Defaults to `"latest_from_last_step"`.
 
         Returns:
             Export information or None if export failed.
@@ -129,6 +138,7 @@ def kili(
             normalized_coordinates=True,
             fmt="kili",
             single_file=bool(single_file),
+            export_type=export_type,
         )
 
     def coco(
@@ -142,6 +152,7 @@ def coco(
         include_sent_back_labels: Optional[bool] = None,
         label_type_in: Optional[list[LabelType]] = None,
         layout: SplitOption = "split",
+        export_type: ExportType = "latest_from_last_step",
     ):
         """Export project labels in COCO format.
 
@@ -165,6 +176,14 @@ def coco(
                 By default, only `DEFAULT` and `REVIEW` labels are exported.
             layout: Layout of the exported files. "split" means there is one folder
                 per job, "merged" that there is one folder with every labels.
+            export_type: Type of export. Options are:
+                - `"latest_from_last_step"`: exports the latest label from the last workflow step
+                  (uses `latestLabels` field, supports multiple annotators).
+                - `"latest_from_all_steps"`: exports latest labels from all workflow steps
+                  (useful for consensus projects with multiple annotators).
+                - `"latest"`: exports the single latest label per asset (uses `latestLabel` field).
+                - `"normal"`: exports all labels for each asset.
+                Defaults to `"latest_from_last_step"`.
 
         Returns:
             Export information or None if export failed.
@@ -180,6 +199,7 @@ def coco(
             label_type_in=label_type_in,
             layout=layout,
             fmt="coco",
+            export_type=export_type,
         )
 
     def yolo_v4(
@@ -192,6 +212,7 @@ def yolo_v4(
         filter: Optional[ExportAssetFilter] = None,
         include_sent_back_labels: Optional[bool] = None,
         label_type_in: Optional[list[LabelType]] = None,
+        export_type: ExportType = "latest_from_last_step",
     ):
         """Export project labels in YOLO v4 format.
 
@@ -214,6 +235,14 @@ def yolo_v4(
                 By default, only `DEFAULT` and `REVIEW` labels are exported.
             include_sent_back_labels: If True, the export will include the labels that
                 have been sent back.
+            export_type: Type of export. Options are:
+                - `"latest_from_last_step"`: exports the latest label from the last workflow step
+                  (uses `latestLabels` field, supports multiple annotators).
+                - `"latest_from_all_steps"`: exports latest labels from all workflow steps
+                  (useful for consensus projects with multiple annotators).
+                - `"latest"`: exports the single latest label per asset (uses `latestLabel` field).
+                - `"normal"`: exports all labels for each asset.
+                Defaults to `"latest_from_last_step"`.
 
         Returns:
             Export information or None if export failed.
@@ -228,6 +257,7 @@ def yolo_v4(
             label_type_in=label_type_in,
             layout=layout,
             fmt="yolo_v4",
+            export_type=export_type,
         )
 
     def yolo_v5(
@@ -240,6 +270,7 @@ def yolo_v5(
         filter: Optional[ExportAssetFilter] = None,
         include_sent_back_labels: Optional[bool] = None,
         label_type_in: Optional[list[LabelType]] = None,
+        export_type: ExportType = "latest_from_last_step",
     ):
         """Export project labels in YOLO v5 format.
 
@@ -262,6 +293,14 @@ def yolo_v5(
                 By default, only `DEFAULT` and `REVIEW` labels are exported.
             include_sent_back_labels: If True, the export will include the labels that
                 have been sent back.
+            export_type: Type of export. Options are:
+                - `"latest_from_last_step"`: exports the latest label from the last workflow step
+                  (uses `latestLabels` field, supports multiple annotators).
+                - `"latest_from_all_steps"`: exports latest labels from all workflow steps
+                  (useful for consensus projects with multiple annotators).
+                - `"latest"`: exports the single latest label per asset (uses `latestLabel` field).
+                - `"normal"`: exports all labels for each asset.
+                Defaults to `"latest_from_last_step"`.
 
         Returns:
             Export information or None if export failed.
@@ -276,6 +315,7 @@ def yolo_v5(
             label_type_in=label_type_in,
             layout=layout,
             fmt="yolo_v5",
+            export_type=export_type,
         )
 
     def yolo_v7(
@@ -288,6 +328,7 @@ def yolo_v7(
         filter: Optional[ExportAssetFilter] = None,
         include_sent_back_labels: Optional[bool] = None,
         label_type_in: Optional[list[LabelType]] = None,
+        export_type: ExportType = "latest_from_last_step",
     ):
         """Export project labels in YOLO v7 format.
 
@@ -310,6 +351,14 @@ def yolo_v7(
                 By default, only `DEFAULT` and `REVIEW` labels are exported.
             include_sent_back_labels: If True, the export will include the labels that
                 have been sent back.
+            export_type: Type of export. Options are:
+                - `"latest_from_last_step"`: exports the latest label from the last workflow step
+                  (uses `latestLabels` field, supports multiple annotators).
+                - `"latest_from_all_steps"`: exports latest labels from all workflow steps
+                  (useful for consensus projects with multiple annotators).
+                - `"latest"`: exports the single latest label per asset (uses `latestLabel` field).
+                - `"normal"`: exports all labels for each asset.
+                Defaults to `"latest_from_last_step"`.
 
         Returns:
             Export information or None if export failed.
@@ -324,6 +373,7 @@ def yolo_v7(
             label_type_in=label_type_in,
             layout=layout,
             fmt="yolo_v7",
+            export_type=export_type,
         )
 
     def yolo_v8(
@@ -336,6 +386,7 @@ def yolo_v8(
         filter: Optional[ExportAssetFilter] = None,
         include_sent_back_labels: Optional[bool] = None,
         label_type_in: Optional[list[LabelType]] = None,
+        export_type: ExportType = "latest_from_last_step",
     ):
         """Export project labels in YOLO v8 format.
 
@@ -358,6 +409,14 @@ def yolo_v8(
                 By default, only `DEFAULT` and `REVIEW` labels are exported.
             include_sent_back_labels: If True, the export will include the labels that
                 have been sent back.
+            export_type: Type of export. Options are:
+                - `"latest_from_last_step"`: exports the latest label from the last workflow step
+                  (uses `latestLabels` field, supports multiple annotators).
+                - `"latest_from_all_steps"`: exports latest labels from all workflow steps
+                  (useful for consensus projects with multiple annotators).
+                - `"latest"`: exports the single latest label per asset (uses `latestLabel` field).
+                - `"normal"`: exports all labels for each asset.
+                Defaults to `"latest_from_last_step"`.
 
         Returns:
             Export information or None if export failed.
@@ -372,6 +431,7 @@ def yolo_v8(
             label_type_in=label_type_in,
             layout=layout,
             fmt="yolo_v8",
+            export_type=export_type,
         )
 
     def pascal_voc(
@@ -383,6 +443,7 @@ def pascal_voc(
         filter: Optional[ExportAssetFilter] = None,
         include_sent_back_labels: Optional[bool] = None,
         label_type_in: Optional[list[LabelType]] = None,
+        export_type: ExportType = "latest_from_last_step",
     ):
         """Export project labels in Pascal VOC format.
 
@@ -403,6 +464,14 @@ def pascal_voc(
                 By default, only `DEFAULT` and `REVIEW` labels are exported.
             include_sent_back_labels: If True, the export will include the labels that
                 have been sent back.
+            export_type: Type of export. Options are:
+                - `"latest_from_last_step"`: exports the latest label from the last workflow step
+                  (uses `latestLabels` field, supports multiple annotators).
+                - `"latest_from_all_steps"`: exports latest labels from all workflow steps
+                  (useful for consensus projects with multiple annotators).
+                - `"latest"`: exports the single latest label per asset (uses `latestLabel` field).
+                - `"normal"`: exports all labels for each asset.
+                Defaults to `"latest_from_last_step"`.
 
         Returns:
             Export information or None if export failed.
@@ -417,6 +486,7 @@ def pascal_voc(
             label_type_in=label_type_in,
             layout="merged",
             fmt="pascal_voc",
+            export_type=export_type,
         )
 
     def geojson(
@@ -428,6 +498,7 @@ def geojson(
         filter: Optional[ExportAssetFilter] = None,
         include_sent_back_labels: Optional[bool] = None,
         label_type_in: Optional[list[LabelType]] = None,
+        export_type: ExportType = "latest_from_last_step",
     ):
         """Export project labels in GeoJSON format.
 
@@ -448,6 +519,14 @@ def geojson(
                 By default, only `DEFAULT` and `REVIEW` labels are exported.
             include_sent_back_labels: If True, the export will include the labels that
                 have been sent back.
+            export_type: Type of export. Options are:
+                - `"latest_from_last_step"`: exports the latest label from the last workflow step
+                  (uses `latestLabels` field, supports multiple annotators).
+                - `"latest_from_all_steps"`: exports latest labels from all workflow steps
+                  (useful for consensus projects with multiple annotators).
+                - `"latest"`: exports the single latest label per asset (uses `latestLabel` field).
+                - `"normal"`: exports all labels for each asset.
+                Defaults to `"latest_from_last_step"`.
 
         Returns:
             Export information or None if export failed.
@@ -462,6 +541,7 @@ def geojson(
             label_type_in=label_type_in,
             layout="merged",
             fmt="geojson",
+            export_type=export_type,
         )
 
     @typechecked
@@ -531,6 +611,7 @@ def _export(
         project_id: str,
         single_file: bool = False,
         with_assets: Optional[bool] = True,
+        export_type: ExportType = "latest_from_last_step",
     ) -> Optional[list[dict[str, Union[list[str], str]]]]:
         """Export the project labels with the requested format into the requested output path.
 
@@ -591,4 +672,5 @@ def _export(
             normalized_coordinates=normalized_coordinates,
             label_type_in=list(label_type_in) if label_type_in else None,
             include_sent_back_labels=include_sent_back_labels,
+            export_type=export_type,
         )

src/kili/presentation/client/label.py

@@ -44,7 +44,7 @@
 )
 from kili.services.export import export_labels
 from kili.services.export.exceptions import NoCompatibleJobError
-from kili.services.export.types import CocoAnnotationModifier, LabelFormat, SplitOption
+from kili.services.export.types import CocoAnnotationModifier, ExportType, LabelFormat, SplitOption
 from kili.use_cases.asset.utils import AssetUseCasesUtils
 from kili.use_cases.label import LabelUseCases
 from kili.use_cases.label.process_shapefiles import get_json_response_from_shapefiles
@@ -1298,6 +1298,7 @@ def export_labels(
         normalized_coordinates: Optional[bool] = None,
         label_type_in: Optional[list[str]] = None,
         include_sent_back_labels: Optional[bool] = None,
+        export_type: ExportType = "latest",
     ) -> Optional[list[dict[str, Union[list[str], str]]]]:
         # pylint: disable=line-too-long
         """Export the project labels with the requested format into the requested output path.
@@ -1355,6 +1356,11 @@ def export_labels(
             label_type_in: Optional list of label type. Exported assets should have a label whose type belongs to that list.
                 By default, only `DEFAULT` and `REVIEW` labels are exported.
             include_sent_back_labels: If True, the export will include the labels that have been sent back.
+            export_type: Type of export. Options are:
+                - `"latest"`: Export only the latest label for each asset (deprecated, use `"latest_from_last_step"` instead).
+                - `"latest_from_last_step"`: Export the latest label from each annotator for the last step.
+                - `"latest_from_all_steps"`: Export the latest label from each annotator for all steps (uses `isLastForStep` filtering).
+                - `"normal"`: Export all labels.
 
         !!! Info
             The supported formats are:
@@ -1435,7 +1441,7 @@ def is_rectangle(coco_annotation, coco_image, kili_annotation):
                 self,  # pyright: ignore[reportArgumentType]
                 asset_ids=resolved_asset_ids,
                 project_id=ProjectId(project_id),
-                export_type="latest",
+                export_type=export_type,
                 label_format=fmt,
                 split_option=layout,
                 single_file=single_file,