Merge branch 'master' of github.com:heartexlabs/label-studio-ml-backend into fix/gliner

Author: makseq

.github/workflows/build.yml

@@ -80,7 +80,7 @@ jobs:
           echo "image_branch_version=$image_branch_version" >> $GITHUB_OUTPUT
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.6.1
+        uses: docker/setup-buildx-action@v3.7.1
 
       - name: Login to DockerHub
         if: ${{ !github.event.pull_request.head.repo.fork }}
@@ -109,7 +109,7 @@ jobs:
             core.setOutput("tags", tags);
 
       - name: Push Docker image
-        uses: docker/build-push-action@v6.7.0
+        uses: docker/build-push-action@v6.9.0
         id: docker_build_and_push
         with:
           context: "${{ env.DOCKER_EXAMPLES_DIRECTORY }}/${{ env.backend_dir_name }}"

.github/workflows/tests.yml

@@ -121,15 +121,15 @@ jobs:
 
       - name: "Upload general coverage to Codecov"
         if: ${{ matrix.backend_dir_name == 'the_simplest_backend' }}
-        uses: codecov/codecov-action@v4.5.0
+        uses: codecov/codecov-action@v4.6.0
         with:
           name: codecov-general
           files: ./tests/${{ matrix.backend_dir_name }}_coverage.xml
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: false
 
       - name: "Upload ml-backend ${{ matrix.backend_dir_name }} coverage to Codecov"
-        uses: codecov/codecov-action@v4.5.0
+        uses: codecov/codecov-action@v4.6.0
         with:
           name: codecov-${{ matrix.backend_dir_name }}
           files: ./label_studio_ml/examples/${{ matrix.backend_dir_name }}/coverage.xml

label_studio_ml/examples/yolo/README.md

@@ -82,7 +82,7 @@ This tutorial uses the [YOLO example](https://github.com/HumanSignal/label-studi
 | `model_classifier_f1_threshold`       | float  | 0.95    | F1 score threshold for early stopping during training. Set to prevent overfitting.                                                                |
 | `model_classifier_accuracy_threshold` | float  | 1.00    | Accuracy threshold for early stopping during training. Set to prevent overfitting.                                                                |
 | `model_score_threshold`               | float  | 0.5     | Minimum confidence threshold for predictions. Labels with confidence below this threshold will be disregarded.                                    |
-| `model_path`                          | string | None    | Path to the custom YOLO model. See more in the section "Custom YOLO Models."                                                                      |
+| `model_path`                          | string | None    | Path to the custom YOLO model. See more in the section [Your own custom YOLO models](./README.md#your-own-custom-yolo-models).                                                                      |
 
 **Note:** You can customize the neural network parameters directly in the labeling configuration by adjusting the attributes in the `<TimelineLabels>` tag.
 
@@ -158,7 +158,7 @@ and it generates predictions for each frame in the video.
 
 #### Custom YOLO models for feature extraction
 
-You can load your own YOLO models using the steps described in the [main README](./README.md#custom-yolo-models).
+You can load your own YOLO models using the steps described in the [main README](./README.md#your-own-custom-yolo-models).
 However, it should have similar architecture as `yolov8-cls` models. See `utils/neural_nets.py::cached_feature_extraction()` for more details.
 
 #### Cache folder 

label_studio_ml/examples/yolo/README_TIMELINE_LABELS.md

@@ -198,4 +198,4 @@ def __str__(self):
 
     class Config:
         arbitrary_types_allowed = True
-        protected_namespaces = ('__.*__', '_.*')  # Excludes 'model_'
+        protected_namespaces = ("__.*__", "_.*")  # Excludes 'model_'

label_studio_ml/examples/yolo/control_models/base.py

@@ -36,7 +36,7 @@ def is_control_matched(cls, control) -> bool:
 
     def build_point_mapping(self):
         """Build a mapping between points and Label Studio labels, e.g.
-        <Label value="left_eye" predicted_values="person" model_index="2" /> => {"person::2": "left_eye"}
+        <Label value="nose" predicted_values="person" model_index="0" /> => {"person::0": "nose"}
         """
         mapping = {}
         for value, label_tag in self.control.labels_attrs.items():
@@ -80,12 +80,15 @@ def create_keypoints(self, results, path):
             )  # Convert normalized keypoints to percentages
             model_label = model_names[int(results[0].boxes.cls[bbox_index])]
 
+            point_logs = "\n".join(
+                [f' model_index="{i}", xy={xyn}' for i, xyn in enumerate(point_xyn)]
+            )
             logger.debug(
                 "----------------------\n"
                 f"task id > {path}\n"
                 f"type: {self.control}\n"
-                f"keypoints > {point_xyn}\n"
                 f"model label > {model_label}\n"
+                f"keypoints >\n{point_logs}\n"
                 f"confidences > {bbox_conf}\n"
             )
 
@@ -115,7 +118,7 @@ def create_keypoints(self, results, path):
                     logger.warning(
                         f"Point {index_name} not found in point map, "
                         f"you have to define it in the labeling config, e.g.:\n"
-                        f'<Label value="nose" predicted_values="person" index="1" />'
+                        f'<Label value="nose" predicted_values="person" model_index="0" />'
                     )
                     continue
                 point_label = self.point_map[index_name]
@@ -126,10 +129,10 @@ def create_keypoints(self, results, path):
                     "to_name": self.to_name,
                     "type": "keypointlabels",
                     "value": {
-                        "keypointlabels": [point_label],  # Keypoint label
-                        "width": self.point_size
-                        / image_width
-                        * 100,  # Keypoint width, just visual styling
+                        # point label
+                        "keypointlabels": [point_label],
+                        # point width, just visual styling
+                        "width": self.point_size / image_width * 100,
                         "x": x,
                         "y": y,
                     },

label_studio_ml/examples/yolo/control_models/keypoint_labels.py

@@ -7,7 +7,7 @@
     BaseNN,
     MultiLabelLSTM,
     cached_feature_extraction,
-    cached_yolo_predict
+    cached_yolo_predict,
 )
 from utils.converter import (
     get_label_map,
@@ -22,7 +22,7 @@
 class TimelineLabelsModel(ControlModel):
     """
     Class representing a TimelineLabels control tag for YOLO model.
-    See README_TIMELINE_LABELS.md for more details. 
+    See README_TIMELINE_LABELS.md for more details.
     """
 
     type = "TimelineLabels"
@@ -50,7 +50,7 @@ def create(cls, *args, **kwargs):
                 f"TimelinesLabels model works in simple mode (without training), "
                 f"but no labels from YOLO model names are matched:\n{instance.control.name}\n"
                 f"Add labels from YOLO model names to the labeling config or use `predicted_values` to map them. "
-                f"As alternative option, you can set `model_trainable=\"true\"` in the TimelineLabels control tag "
+                f'As alternative option, you can set `model_trainable="true"` in the TimelineLabels control tag '
                 f"to train the model on the labels from the labeling config."
             )
         return instance
@@ -64,15 +64,21 @@ def predict_regions(self, video_path) -> List[Dict]:
     def create_timelines_simple(self, video_path):
         logger.debug(f"create_timelines_simple: {self.from_name}")
         # get yolo predictions
-        frame_results = cached_yolo_predict(self.model, video_path, self.model.model_name)
+        frame_results = cached_yolo_predict(
+            self.model, video_path, self.model.model_name
+        )
 
         # Initialize a dictionary to keep track of ongoing segments for each label
         model_names = self.model.names
         needed_ids = [i for i, name in model_names.items() if name in self.label_map]
-        needed_labels = [name for i, name in model_names.items() if name in self.label_map]
+        needed_labels = [
+            name for i, name in model_names.items() if name in self.label_map
+        ]
 
         probs = [frame.probs.data[needed_ids].cpu().numpy() for frame in frame_results]
-        label_map = {self.label_map[label]: idx for idx, label in enumerate(needed_labels)}
+        label_map = {
+            self.label_map[label]: idx for idx, label in enumerate(needed_labels)
+        }
 
         return convert_probs_to_timelinelabels(
             probs, label_map, self.control.name, self.model_score_threshold
@@ -81,7 +87,9 @@ def create_timelines_simple(self, video_path):
     def create_timelines_trainable(self, video_path):
         logger.debug(f"create_timelines_trainable: {self.from_name}")
         # extract features based on pre-trained yolo classification model
-        frame_results = cached_feature_extraction(self.model, video_path, self.model.model_name)
+        frame_results = cached_feature_extraction(
+            self.model, video_path, self.model.model_name
+        )
 
         yolo_probs = [frame.probs for frame in frame_results]
         path = self.get_classifier_path(self.project_id)
@@ -95,12 +103,22 @@ def create_timelines_trainable(self, video_path):
         # run predict and convert to timelinelabels
         probs = classifier.predict(yolo_probs)
         regions = convert_probs_to_timelinelabels(
-            probs, classifier.get_label_map(), self.control.name, self.model_score_threshold
+            probs,
+            classifier.get_label_map(),
+            self.control.name,
+            self.model_score_threshold,
         )
 
         return regions
 
     def fit(self, event, data, **kwargs):
+        if not self.trainable:
+            logger.debug(
+                'TimelineLabels model is in not trainable mode. '
+                'Use model_trainable="true" to enable training.'
+            )
+            return
+
         """Fit the model."""
         if event == "START_TRAINING":
             # TODO: the full training makes a lot of sense here, but it's not implemented yet
@@ -109,17 +127,20 @@ def fit(self, event, data, **kwargs):
             )
 
         if event in ("ANNOTATION_CREATED", "ANNOTATION_UPDATED"):
-            features, labels, label_map, project_id = self.load_features_and_labels(data)
+            features, labels, label_map, project_id = self.load_features_and_labels(
+                data
+            )
             classifier, path = self.load_classifier(features, label_map, project_id)
             return self.train_classifier(classifier, features, labels, path)
 
     def train_classifier(self, classifier, features, labels, path):
-        """ Train the classifier model for timelinelabels using incremental partial learning.
-        """
+        """Train the classifier model for timelinelabels using incremental partial learning."""
         # Stop training when accuracy or f1 score reaches this threshold, it helps to avoid overfitting
         # because we partially train it on a small dataset from one annotation only
         get = self.control.attr.get
-        epochs = int(get("model_classifier_epochs", 1000))  # Maximum number of training epochs
+        epochs = int(
+            get("model_classifier_epochs", 1000)
+        )  # Maximum number of training epochs
         f1_threshold = float(get("model_classifier_f1_threshold", 0.95))
         accuracy_threshold = float(get("model_classifier_accuracy_threshold", 1.00))
 
@@ -129,13 +150,13 @@ def train_classifier(self, classifier, features, labels, path):
             labels,
             epochs=epochs,
             f1_threshold=f1_threshold,
-            accuracy_threshold=accuracy_threshold
+            accuracy_threshold=accuracy_threshold,
         )
         classifier.save_and_cache(path)
         return result
 
     def load_classifier(self, features, label_map, project_id):
-        """ Load or create a classifier model for timelinelabels.
+        """Load or create a classifier model for timelinelabels.
         1. Load neural network parameters from labeling config.
         2. Try loading classifier model from memory cache, then from disk.
         3. Or create a new classifier instance if there wasn't successful loading, or if parameters have changed.
@@ -155,11 +176,11 @@ def load_classifier(self, features, label_map, project_id):
         # Create a new classifier instance if it doesn't exist
         # or if labeling config has changed
         if (
-                not classifier
-                or classifier.label_map != label_map
-                or classifier.sequence_size != sequence_size
-                or classifier.hidden_size != hidden_size
-                or classifier.num_layers != num_layers
+            not classifier
+            or classifier.label_map != label_map
+            or classifier.sequence_size != sequence_size
+            or classifier.hidden_size != hidden_size
+            or classifier.num_layers != num_layers
         ):
             logger.info("Creating a new classifier model for timelinelabels")
             input_size = len(features[0])
@@ -176,7 +197,7 @@ def load_classifier(self, features, label_map, project_id):
         return classifier, path
 
     def load_features_and_labels(self, data):
-        """ Load features and labels from the annotation
+        """Load features and labels from the annotation
         Args:
             data: event data, dictionary with keys 'task' and 'annotation'
         Returns:

label_studio_ml/examples/yolo/control_models/timeline_labels.py

@@ -74,7 +74,9 @@ def predict_regions(self, path) -> List[Dict]:
 
         # run model track
         try:
-            results = self.model.track(path, conf=conf, iou=iou, tracker=tracker, stream=True)
+            results = self.model.track(
+                path, conf=conf, iou=iou, tracker=tracker, stream=True
+            )
         finally:
             # clean temporary file
             if tmp_yaml and os.path.exists(tmp_yaml):

label_studio_ml/examples/yolo/control_models/video_rectangle.py

@@ -30,7 +30,14 @@ def test_multi_label_lstm():
     ).tolist()  # Shape: (batch_size, seq_len, output_size)
 
     # Perform partial training with batch size of 16
-    model.partial_fit(data, labels, batch_size=16, epochs=1000, accuracy_threshold=0.999, f1_threshold=0.999)
+    model.partial_fit(
+        data,
+        labels,
+        batch_size=16,
+        epochs=1000,
+        accuracy_threshold=0.999,
+        f1_threshold=0.999,
+    )
 
     # Example prediction
     predictions = model.predict(data)