Merge branch 'main' into feat/expose-stream-metrics-prometheus

Author: PawelPeczek-Roboflow

docs/workflows/execution_engine_changelog.md

@@ -2,6 +2,80 @@
 
 Below you can find the changelog for Execution Engine.
 
+## Execution Engine `v1.8.0` | inference `v1.1.1`
+
+!!! Note "Additive change + one breaking change due to bug fix with minimal expected impact"
+
+    This release extends the Execution Engine so that steps gated by control flow (e.g. after a
+    `ContinueIf` block) can run even when they have **no data-derived lineage** — i.e. when they
+    do not receive batch-oriented inputs from upstream steps. Lineage and execution dimensionality
+    can now be derived from control flow predecessor steps. Existing workflows are unaffected.
+    
+    One breaking change introduced is due to the bug fix that affects `Batch.remove_by_indices` with nested batches (see below); impact is
+    expected to be minimal.
+
+**What changed**
+
+* **Control flow lineage** — The compiler now tracks lineage that comes from control flow steps
+  (e.g. branches after `ContinueIf`). A new notion of **control flow lineage support** is used when
+  a step has no batch-oriented data inputs but is preceded by control flow steps: the step’s
+  execution slices and batch structure are taken from those control flow predecessors.
+
+* **Loosened compatibility check** — Previously, `verify_compatibility_of_input_data_lineage_with_control_flow_lineage`
+  raised `ControlFlowDefinitionError` for any step that had control flow predecessors but no
+  data-derived lineage, so such steps could not be compiled. That check is now relaxed: when a step
+  has no input data lineage, compatibility is not enforced and the step’s lineage is derived from
+  the control flow predecessor step lineage instead. The strict check still runs when the step
+  *does* have data-derived lineage, to ensure control flow and data lineage remain compatible.
+
+* **New step patterns** — Steps that are triggered only by control flow and do not consume
+  batch data now run correctly. For example, you can send email notifications (or run other
+  side-effect steps) after a `ContinueIf` without wiring any data into parameters like
+  `message_parameters`; the step will execute once per control flow branch with lineage and
+  dimensionality taken from the controlling step.
+
+* **`Batch.remove_by_indices` with nested batches (behavioral fix)** — When removing indices
+  via `Batch.remove_by_indices`, nested `Batch` elements are now recursively filtered by the
+  same index set. As a result, entries at removed indices (including `None` values) are now
+  correctly dropped from nested batches as well. Previously, only the top-level batch was
+  filtered; nested batches were left unchanged.
+  
+  By default for a `WorkflowBlock`, `accepts_empty_values()`is `False`. While
+  this was bypassed, blocks consuming such inputs where outright failing as for example `StitchDetectionsBatchBlock`:
+
+    ```python
+    def run(
+        self,
+        images: Batch[WorkflowImageData],
+        images_predictions: Batch[Batch[sv.Detections]],
+    ) -> BlockResult:
+        result = []
+        for image, image_predictions in zip(images, images_predictions):
+            image_predictions = [deepcopy(p) for p in image_predictions if len(p)]
+            for p in image_predictions:
+                coords = p["parent_coordinates"][0]
+        ...
+    ```
+
+  The only core block that this change affects is the `DimensionCollapseBlockV1` block,
+  As it was wrapping individual inputs in a batch without filtering for None values.
+
+  ```python
+  class DimensionCollapseBlockV1(WorkflowBlock):
+
+    @classmethod
+    def get_manifest(cls) -> Type[WorkflowBlockManifest]:
+        return BlockManifest
+
+    def run(self, data: Batch[Any]) -> BlockResult:
+        return {"output": [e for e in data]}
+  ```
+  
+  When using the output from this block downstream applications could either outright fail or
+  silently process None values, unless they filtered those values themselves.
+
+  Given that above we reckon the impact will be minimal.
+
 ## Execution Engine `v1.7.0` | inference `v0.59.0`
 
 !!! warning "Breaking change regarding step errors in workflows"

inference/core/env.py

@@ -204,6 +204,8 @@
 
 QWEN_3_ENABLED = str2bool(os.getenv("QWEN_3_ENABLED", True))
 
+QWEN_3_5_ENABLED = str2bool(os.getenv("QWEN_3_5_ENABLED", True))
+
 DEPTH_ESTIMATION_ENABLED = str2bool(os.getenv("DEPTH_ESTIMATION_ENABLED", True))
 
 SMOLVLM2_ENABLED = str2bool(os.getenv("SMOLVLM2_ENABLED", True))

inference/core/managers/active_learning.py

@@ -72,6 +72,8 @@ def register(
     ) -> None:
         try:
             resolved_model_id = resolve_roboflow_model_alias(model_id=model_id)
+            if not hasattr(request, "active_learning_target_dataset"):
+                return None
             target_dataset = (
                 request.active_learning_target_dataset
                 or resolved_model_id.split("/")[0]

inference/core/models/inference_models_adapters.py

@@ -1,8 +1,11 @@
+import base64
+import io
 from io import BytesIO
 from time import perf_counter
 from typing import Any, List, Optional, Tuple, Union
 
 import numpy as np
+import torch
 from PIL import Image, ImageDraw, ImageFont
 
 from inference.core.entities.requests import (
@@ -22,6 +25,8 @@
     ObjectDetectionInferenceResponse,
     ObjectDetectionPrediction,
     Point,
+    SemanticSegmentationInferenceResponse,
+    SemanticSegmentationPrediction,
 )
 from inference.core.env import (
     ALLOW_INFERENCE_MODELS_DIRECTLY_ACCESS_LOCAL_PACKAGES,
@@ -48,6 +53,10 @@
     MultiLabelClassificationModel,
     MultiLabelClassificationPrediction,
     ObjectDetectionModel,
+    SemanticSegmentationModel,
+)
+from inference_models.models.base.semantic_segmentation import (
+    SemanticSegmentationResult,
 )
 from inference_models.models.base.types import PreprocessingMetadata
 
@@ -855,3 +864,133 @@ def draw_predictions(inference_request, inference_response, class_names: List[st
     image = image.convert("RGB")
     image.save(buffered, format="JPEG")
     return buffered.getvalue()
+
+
+class InferenceModelsSemanticSegmentationAdapter(Model):
+    def __init__(self, model_id: str, api_key: str = None, **kwargs):
+        super().__init__()
+
+        self.metrics = {"num_inferences": 0, "avg_inference_time": 0.0}
+
+        self.api_key = api_key if api_key else API_KEY
+        model_id = resolve_roboflow_model_alias(model_id=model_id)
+
+        self.task_type = "semantic-segmentation"
+
+        extra_weights_provider_headers = get_extra_weights_provider_headers(
+            countinference=kwargs.get("countinference"),
+            service_secret=kwargs.get("service_secret"),
+        )
+        backend = list(
+            VALID_INFERENCE_MODELS_BACKENDS.difference(
+                DISABLED_INFERENCE_MODELS_BACKENDS
+            )
+        )
+        self._model: SemanticSegmentationModel = AutoModel.from_pretrained(
+            model_id_or_path=model_id,
+            api_key=self.api_key,
+            allow_untrusted_packages=ALLOW_INFERENCE_MODELS_UNTRUSTED_PACKAGES,
+            allow_direct_local_storage_loading=ALLOW_INFERENCE_MODELS_DIRECTLY_ACCESS_LOCAL_PACKAGES,
+            weights_provider_extra_headers=extra_weights_provider_headers,
+            backend=backend,
+            **kwargs,
+        )
+        self.class_names = list(self._model.class_names)
+
+    @property
+    def class_map(self):
+        # match segment.roboflow.com
+        return {str(k): v for k, v in enumerate(self.class_names)}
+
+    def map_inference_kwargs(self, kwargs: dict) -> dict:
+        return kwargs
+
+    def preprocess(self, image: Any, **kwargs):
+        is_batch = isinstance(image, list)
+        images = image if is_batch else [image]
+        np_images: List[np.ndarray] = [
+            load_image_bgr(
+                v,
+                disable_preproc_auto_orient=kwargs.get(
+                    "disable_preproc_auto_orient", False
+                ),
+            )
+            for v in images
+        ]
+        mapped_kwargs = self.map_inference_kwargs(kwargs)
+        return self._model.pre_process(np_images, **mapped_kwargs)
+
+    def predict(self, img_in, **kwargs):
+        mapped_kwargs = self.map_inference_kwargs(kwargs)
+        return self._model.forward(img_in, **mapped_kwargs)
+
+    def postprocess(
+        self,
+        predictions: torch.Tensor,
+        preprocess_return_metadata: PreprocessingMetadata,
+        **kwargs,
+    ) -> List[SemanticSegmentationInferenceResponse]:
+        mapped_kwargs = self.map_inference_kwargs(kwargs)
+        segmentation_results = self._model.post_process(
+            predictions, preprocess_return_metadata, **mapped_kwargs
+        )
+
+        responses: List[SemanticSegmentationInferenceResponse] = []
+        for preproc_metadata, segmentation in zip(
+            preprocess_return_metadata, segmentation_results
+        ):
+            height = preproc_metadata.original_size.height
+            width = preproc_metadata.original_size.width
+            response_image = InferenceResponseImage(width=width, height=height)
+            # WARNING! This way of conversion is hazardous - first of all, if background class is not in class names,
+            # for certain pre-processing, we end up with -1 values which will be wrapped to 255 - second of all,
+            # we can support only 256 classes - those constraints should be fine until inference 2.0
+            response_predictions = SemanticSegmentationPrediction(
+                segmentation_mask=self.img_to_b64_str(
+                    segmentation.segmentation_map.to(torch.uint8)
+                ),
+                confidence_mask=self.img_to_b64_str(
+                    (segmentation.confidence * 255).to(torch.uint8)
+                ),
+                class_map=self.class_map,
+                image=dict(response_image),
+            )
+            response = SemanticSegmentationInferenceResponse(
+                predictions=response_predictions,
+                image=response_image,
+            )
+            responses.append(response)
+        return responses
+
+    def clear_cache(self, delete_from_disk: bool = True) -> None:
+        """Clears any cache if necessary. TODO: Implement this to delete the cache from the experimental model.
+
+        Args:
+            delete_from_disk (bool, optional): Whether to delete cached files from disk. Defaults to True.
+        """
+        pass
+
+    def img_to_b64_str(self, img: torch.Tensor) -> str:
+        if img.dtype != torch.uint8:
+            raise ValueError(
+                f"img_to_b64_str requires uint8 tensor but got dtype {img.dtype}"
+            )
+
+        img = Image.fromarray(img.cpu().numpy())
+        buffered = io.BytesIO()
+        img.save(buffered, format="PNG")
+
+        img_str = base64.b64encode(buffered.getvalue())
+        img_str = img_str.decode("ascii")
+
+        return img_str
+
+    def draw_predictions(
+        self,
+        inference_request: InferenceRequest,
+        inference_response: InferenceResponse,
+    ) -> bytes:
+        raise NotImplementedError(
+            "draw_predictions(...) is not implemented for semantic segmentation models - responses contain "
+            "visualization already."
+        )

inference/core/models/semantic_segmentation_base.py

@@ -134,7 +134,7 @@ def img_to_b64_str(self, img: torch.Tensor) -> str:
                 f"img_to_b64_str requires uint8 tensor but got dtype {img.dtype}"
             )
 
-        img = Image.fromarray(img.numpy())
+        img = Image.fromarray(img.cpu().numpy())
         buffered = io.BytesIO()
         img.save(buffered, format="PNG")
 

inference/core/registries/roboflow.py

@@ -19,6 +19,7 @@
     MODELS_CACHE_AUTH_CACHE_MAX_SIZE,
     MODELS_CACHE_AUTH_CACHE_TTL,
     MODELS_CACHE_AUTH_ENABLED,
+    USE_INFERENCE_MODELS,
 )
 from inference.core.exceptions import (
     MissingApiKeyError,
@@ -34,6 +35,7 @@
     MODEL_TYPE_KEY,
     PROJECT_TASK_TYPE_KEY,
     ModelEndpointType,
+    get_model_metadata_from_inference_models_registry,
     get_roboflow_dataset_type,
     get_roboflow_instant_model_data,
     get_roboflow_model_data,
@@ -129,13 +131,20 @@ def _check_if_api_key_has_access_to_model(
                 countinference=countinference,
                 service_secret=service_secret,
             )
-        else:
+        elif not USE_INFERENCE_MODELS:
             get_roboflow_instant_model_data(
                 api_key=api_key,
                 model_id=model_id,
                 countinference=countinference,
                 service_secret=service_secret,
             )
+        else:
+            get_model_metadata_from_inference_models_registry(
+                api_key=api_key,
+                model_id=model_id,
+                countinference=countinference,
+                service_secret=service_secret,
+            )
     except RoboflowAPINotAuthorizedError:
         return False
     return True
@@ -220,14 +229,22 @@ def get_model_type(
             device_id=GLOBAL_DEVICE_ID,
         ).get("ort")
         project_task_type = api_data.get("type", "object-detection")
-    else:
+    elif not USE_INFERENCE_MODELS:
         api_data = get_roboflow_instant_model_data(
             api_key=api_key,
             model_id=model_id,
             countinference=countinference,
             service_secret=service_secret,
         )
         project_task_type = api_data.get("taskType", "object-detection")
+    else:
+        api_data = get_model_metadata_from_inference_models_registry(
+            api_key=api_key,
+            model_id=model_id,
+            countinference=countinference,
+            service_secret=service_secret,
+        )
+        project_task_type = api_data.get("taskType", "object-detection")
     if api_data is None:
         raise ModelArtefactError("Error loading model artifacts from Roboflow API.")