v0.37.0

See https://github.com/quic/ai-hub-models/releases/v0.37.0 for changelog.

Signed-off-by: QAIHM Team <[email protected]
>

Author: qaihm-bot

README.md

@@ -171,6 +171,7 @@ and many more.
 | [ResNet101](https://aihub.qualcomm.com/models/resnet101) | [qai_hub_models.models.resnet101](qai_hub_models/models/resnet101/README.md) |
 | [ResNet18](https://aihub.qualcomm.com/models/resnet18) | [qai_hub_models.models.resnet18](qai_hub_models/models/resnet18/README.md) |
 | [ResNet50](https://aihub.qualcomm.com/models/resnet50) | [qai_hub_models.models.resnet50](qai_hub_models/models/resnet50/README.md) |
+| [Sequencer2D](https://aihub.qualcomm.com/models/sequencer2d) | [qai_hub_models.models.sequencer2d](qai_hub_models/models/sequencer2d/README.md) |
 | [Shufflenet-v2](https://aihub.qualcomm.com/models/shufflenet_v2) | [qai_hub_models.models.shufflenet_v2](qai_hub_models/models/shufflenet_v2/README.md) |
 | [SqueezeNet-1.1](https://aihub.qualcomm.com/models/squeezenet1_1) | [qai_hub_models.models.squeezenet1_1](qai_hub_models/models/squeezenet1_1/README.md) |
 | [Swin-Base](https://aihub.qualcomm.com/models/swin_base) | [qai_hub_models.models.swin_base](qai_hub_models/models/swin_base/README.md) |
@@ -310,6 +311,7 @@ and many more.
 | **Text Generation**
 | [ALLaM-7B](https://aihub.qualcomm.com/models/allam_7b) | [qai_hub_models.models.allam_7b](qai_hub_models/models/allam_7b/README.md) |
 | [Baichuan2-7B](https://aihub.qualcomm.com/models/baichuan2_7b) | [qai_hub_models.models.baichuan2_7b](qai_hub_models/models/baichuan2_7b/README.md) |
+| [Falcon3-7B-Instruct](https://aihub.qualcomm.com/models/falcon_v3_7b_instruct) | [qai_hub_models.models.falcon_v3_7b_instruct](qai_hub_models/models/falcon_v3_7b_instruct/README.md) |
 | [IBM-Granite-v3.1-8B-Instruct](https://aihub.qualcomm.com/models/ibm_granite_v3_1_8b_instruct) | [qai_hub_models.models.ibm_granite_v3_1_8b_instruct](qai_hub_models/models/ibm_granite_v3_1_8b_instruct/README.md) |
 | [IndusQ-1.1B](https://aihub.qualcomm.com/models/indus_1b) | [qai_hub_models.models.indus_1b](qai_hub_models/models/indus_1b/README.md) |
 | [JAIS-6p7b-Chat](https://aihub.qualcomm.com/models/jais_6p7b_chat) | [qai_hub_models.models.jais_6p7b_chat](qai_hub_models/models/jais_6p7b_chat/README.md) |

qai_hub_models/_version.py

@@ -3,4 +3,4 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # ---------------------------------------------------------------------
 
-__version__ = "0.36.0"
+__version__ = "0.37.0"

qai_hub_models/configs/_info_yaml_enums.py

@@ -115,6 +115,7 @@ class MODEL_LICENSE(Enum):
     LLAMA2 = "llama2"
     LLAMA3 = "llama3"
     TAIDE = "taide"
+    FALCON3 = "falcon3"
 
     @property
     def is_copyleft(self) -> bool:
@@ -127,6 +128,7 @@ def is_copyleft(self) -> bool:
             MODEL_LICENSE.LLAMA2,
             MODEL_LICENSE.LLAMA3,
             MODEL_LICENSE.TAIDE,
+            MODEL_LICENSE.FALCON3,
         ]
 
     @property
@@ -169,6 +171,8 @@ def url(self) -> str | None:
             return "https://github.com/facebookresearch/llama/blob/main/LICENSE"
         elif self == MODEL_LICENSE.TAIDE:
             return "https://en.taide.tw/download.html"
+        elif self == MODEL_LICENSE.FALCON3:
+            return "https://falconllm.tii.ae/falcon-terms-and-conditions.html"
         return None
 
 
@@ -206,6 +210,8 @@ class MODEL_USE_CASE(Enum):
     SUPER_RESOLUTION = "Super Resolution"
     SEMANTIC_SEGMENTATION = "Semantic Segmentation"
     DEPTH_ESTIMATION = "Depth Estimation"
+    GAZE_ESTIMATION = "Gaze Estimation"
+
     # Ex: OCR, image caption
     IMAGE_TO_TEXT = "Image To Text"
     OBJECT_DETECTION = "Object Detection"
@@ -234,6 +240,8 @@ def map_to_hf_pipeline_tag(self):
             return "image-segmentation"
         if self.name == "POSE_ESTIMATION":
             return "keypoint-detection"
+        if self.name == "GAZE_ESTIMATION":
+            return "gaze-estimation"
         if self.name == "AUDIO_ENHANCEMENT":
             return "audio-to-audio"
         if self.name == "VIDEO_GENERATION":

qai_hub_models/configs/code_gen_yaml.py

@@ -94,6 +94,10 @@ class QAIHMModelCodeGen(BaseQAIHMConfig):
     # ("AOT prepare") are enabled, both in CI and in Scorecard.
     requires_aot_prepare: bool = False
 
+    # Supported GenAI based runtimes.
+    # If set, ONLY these runtimes will be supported. All others will be disabled.
+    supported_genai_runtimes: list[TargetRuntime] = Field(default_factory=list)
+
     # If set, disables generating `export.py`.
     skip_export: bool = False
 
@@ -171,6 +175,12 @@ def failure_reason(
         """
         Return the reason a model failed or None if the model did not fail.
         """
+        if self.supported_genai_runtimes:
+            if runtime not in self.supported_genai_runtimes:
+                return f"{runtime} is not supported for this GenAI model."
+        elif runtime.is_exclusively_for_genai:
+            return "GenAI runtimes are not supported by this model."
+
         if self.is_precompiled and runtime != TargetRuntime.QNN_CONTEXT_BINARY:
             return "Precompiled models are only supported via the QNN path."
 
@@ -180,7 +190,11 @@ def failure_reason(
         if self.requires_aot_prepare and not runtime.is_aot_compiled:
             return "Only runtimes that are compiled to context binary ahead of time are supported."
 
-        if not self.requires_aot_prepare and runtime.is_aot_compiled:
+        if (
+            not self.requires_aot_prepare
+            and runtime.is_aot_compiled
+            and not runtime.is_exclusively_for_genai
+        ):
             # Only the JIT path is tested if this model does not require AOT prepare.
             # All AOT paths will fail if QNN fails.
             runtime = TargetRuntime.QNN_DLC
@@ -281,6 +295,11 @@ def check_fields(self) -> QAIHMModelCodeGen:
             raise ValueError(
                 "If pip_pre_build_reqs is set, global_requirements_incompatible must also be true."
             )
+        for x in self.supported_genai_runtimes:
+            if not x.is_exclusively_for_genai:
+                raise ValueError(
+                    f"{x.value} is not a GenAI runtime, and should not be listed in supported_genai_runtimes."
+                )
 
         return self
 

qai_hub_models/configs/tool_versions.py

@@ -8,12 +8,7 @@
 from qai_hub.client import JobType
 from qai_hub.public_rest_api import get_job_results
 
-from qai_hub_models.models.common import (
-    InferenceEngine,
-    Optional,
-    QAIRTVersion,
-    TargetRuntime,
-)
+from qai_hub_models.models.common import Optional, QAIRTVersion, TargetRuntime
 from qai_hub_models.utils.base_config import BaseQAIHMConfig
 from qai_hub_models.utils.qai_hub_helpers import extract_job_options
 
@@ -51,7 +46,10 @@ def from_compiled_model(
         Raises:
             ValueError if the model was not compiled by AI Hub.
         """
-        if model.producer is None or not model.producer._job_type == JobType.COMPILE:
+        if model.producer is None or model.producer._job_type not in [
+            JobType.COMPILE,
+            JobType.LINK,
+        ]:
             raise ValueError(
                 "Model must be compiled with AI Hub to extract tool versions."
             )
@@ -117,12 +115,26 @@ def from_job(job: hub.Job, parse_version_tags: bool = False) -> "ToolVersions":
             ValueError if the job type is invalid.
         """
         # Use job_type instead of isinstance to support test mocking.
-        if job._job_type not in [JobType.COMPILE, JobType.PROFILE, JobType.INFERENCE]:
+        if job._job_type not in [
+            JobType.COMPILE,
+            JobType.LINK,
+            JobType.PROFILE,
+            JobType.INFERENCE,
+        ]:
             raise ValueError(
                 f"Cannot extract QAIRT SDK version from job of type {job.job_type}"
             )
 
         if not job.get_status().success:
+            if job._job_type == JobType.LINK:
+                # Link jobs inherit their QAIRT version from input model files.
+                models = cast(hub.LinkJob, job).models
+                for model in models:
+                    if model.producer is not None:
+                        return ToolVersions.from_compiled_model(model)
+                # None of the source models came from us, so we can't detect what QAIRT version to use.
+                return ToolVersions()
+
             # If the job is not successful, the only way to get the QAIRT version is to look at the job flags.
             job_options = extract_job_options(job)
             version: Optional[str] = None
@@ -135,10 +147,7 @@ def from_job(job: hub.Job, parse_version_tags: bool = False) -> "ToolVersions":
                     # QAIRT is applicable for compile jobs only if the target runtime uses QAIRT converters.
                     if x := job_options.get("target_runtime"):
                         rts = [rt for rt in TargetRuntime if rt.value == x]
-                        if (
-                            len(rts) == 1
-                            and rts[0].inference_engine == InferenceEngine.QNN
-                        ):
+                        if len(rts) == 1 and rts[0].qairt_version_changes_compilation:
                             version = "default"
                 else:
                     version = "default"
@@ -157,7 +166,7 @@ def from_job(job: hub.Job, parse_version_tags: bool = False) -> "ToolVersions":
                 qairt=QAIRTVersion(version, validate_exists_on_ai_hub=False)
             )
 
-        if job._job_type == JobType.COMPILE:
+        if job._job_type == JobType.COMPILE or job._job_type == JobType.LINK:
             return ToolVersions.from_compiled_model(
                 cast(hub.Model, cast(hub.CompileJob, job).get_target_model())
             )

qai_hub_models/datasets/__init__.py

@@ -20,25 +20,26 @@
 # We don't want to require a user to install requirements for all datasets just to
 # import the datasets folder. Therefore we only include the datasets that can
 # be imported.
-def _try_import_dataset(module_name: str, cls: str):
+def _try_import_dataset(module_name: str, cls: str, name: str | None = None):
     """
     Import the dataset and add it to the DATASET_NAME_MAP, or pass
     if dependencies for the dataset aren't installed.
     """
+    if name is None:
+        assert module_name[0] == "."
+        name = module_name[1:]
     try:
         module = importlib.import_module(module_name, package="qai_hub_models.datasets")
     except NotImplementedError as e:
         if "AIMET-ONNX" in str(e):
             # stable diffusion dataset requires aimet-onnx
-            _ALL_DATASETS_IMPORT_ERRORS[module_name] = e
+            _ALL_DATASETS_IMPORT_ERRORS[name] = e
             return
         raise e
     except Exception as e:
-        if module_name.startswith("."):
-            module_name = module_name[1:]
         if (
             isinstance(e, ModuleNotFoundError)
-            and str(e) == f"No module named 'qai_hub_models.datasets.{module_name}"
+            and str(e) == f"No module named 'qai_hub_models.datasets{module_name}"
         ):
             # this module legitimately does not exist
             raise e
@@ -48,12 +49,15 @@ def _try_import_dataset(module_name: str, cls: str):
         # By default, the name of the dataset is the name of its module.
         # We add it to this import errors list to hopefully raise the error
         # at a later time (when the user requests this dataset).
-        _ALL_DATASETS_IMPORT_ERRORS[module_name] = e
+        _ALL_DATASETS_IMPORT_ERRORS[name] = e
         return
 
     if x := getattr(module, cls, None):
         xds = cast(type[BaseDataset], x)
-        DATASET_NAME_MAP[xds.dataset_name()] = xds
+        assert (
+            name == xds.dataset_name()
+        ), f"Name is not consistent with call to dataset_name(): {name} vs. {xds.dataset_name()}"
+        DATASET_NAME_MAP[name] = xds
     else:
         raise ValueError(
             f"Could not import {cls}. {cls} was not found in {module_name}"
@@ -70,7 +74,7 @@ def _try_import_dataset(module_name: str, cls: str):
 _try_import_dataset(".coco91class", "Coco91ClassDataset")
 _try_import_dataset(".coco_face", "CocoFaceDataset")
 _try_import_dataset(".human_faces", "HumanFacesDataset")
-_try_import_dataset(".human_faces", "HumanFaces192Dataset")
+_try_import_dataset(".human_faces", "HumanFaces192Dataset", name="human_faces_192")
 _try_import_dataset(".coco_panoptic_seg", "CocoPanopticSegmentationDataset")
 _try_import_dataset(".foot_track_dataset", "FootTrackDataset")
 _try_import_dataset(".gear_guard_dataset", "GearGuardDataset")
@@ -98,16 +102,47 @@ def _try_import_dataset(module_name: str, cls: str):
 _try_import_dataset(".eg1800", "eg1800SegmentationDataset")
 _try_import_dataset(".kitti", "KittiDataset")
 _try_import_dataset(".semantic_kitti", "SemanticKittiDataset")
-_try_import_dataset(".stable_diffusion_calib", "StableDiffusionCalibDatasetTextEncoder")
-_try_import_dataset(".stable_diffusion_calib", "StableDiffusionCalibDatasetUnet")
-_try_import_dataset(".stable_diffusion_calib", "StableDiffusionCalibDatasetVae")
-_try_import_dataset(".stable_diffusion_calib", "StableDiffusionCalibDatasetControlNet")
+_try_import_dataset(
+    ".stable_diffusion_calib",
+    "StableDiffusionCalibDatasetTextEncoder",
+    name="stable_diffusion_calib_text_encoder",
+)
+_try_import_dataset(
+    ".stable_diffusion_calib",
+    "StableDiffusionCalibDatasetUnet",
+    name="stable_diffusion_calib_unet",
+)
+_try_import_dataset(
+    ".stable_diffusion_calib",
+    "StableDiffusionCalibDatasetVae",
+    name="stable_diffusion_calib_vae",
+)
+_try_import_dataset(
+    ".stable_diffusion_calib",
+    "StableDiffusionCalibDatasetControlNet",
+    name="stable_diffusion_calib_controlnet",
+)
 _try_import_dataset(".celebahq", "CelebAHQDataset")
 _try_import_dataset(".wikitext", "WikiText")
 _try_import_dataset(".wikitext_ja", "WikiText_Japanese")
 _try_import_dataset(".tiny_mmlu", "TinyMMLU")
 _try_import_dataset(".mmlu", "MMLU")
 _try_import_dataset(".mmmlu", "MMMLU")
+_try_import_dataset(".mmmlu", "MMMLU_AR", name="mmmlu_ar")
+_try_import_dataset(".mmmlu", "MMMLU_BN", name="mmmlu_bn")
+_try_import_dataset(".mmmlu", "MMMLU_DE", name="mmmlu_de")
+_try_import_dataset(".mmmlu", "MMMLU_ES", name="mmmlu_es")
+_try_import_dataset(".mmmlu", "MMMLU_FR", name="mmmlu_fr")
+_try_import_dataset(".mmmlu", "MMMLU_HI", name="mmmlu_hi")
+_try_import_dataset(".mmmlu", "MMMLU_ID", name="mmmlu_id")
+_try_import_dataset(".mmmlu", "MMMLU_IT", name="mmmlu_it")
+_try_import_dataset(".mmmlu", "MMMLU_JA", name="mmmlu_ja")
+_try_import_dataset(".mmmlu", "MMMLU_KO", name="mmmlu_ko")
+_try_import_dataset(".mmmlu", "MMMLU_PT", name="mmmlu_pt")
+_try_import_dataset(".mmmlu", "MMMLU_SW", name="mmmlu_sw")
+_try_import_dataset(".mmmlu", "MMMLU_YO", name="mmmlu_yo")
+_try_import_dataset(".mmmlu", "MMMLU_ZH", name="mmmlu_zh")
+_try_import_dataset(".mpiigaze", "MPIIGazeDataset")
 _try_import_dataset(".libri_speech", "LibriSpeechDataset")
 _try_import_dataset(
     ".amazon_counterfactual", "AmazonCounterfactualClassificationDataset"

qai_hub_models/datasets/common.py

@@ -10,12 +10,13 @@
 import shutil
 from abc import ABC, abstractmethod
 from collections.abc import Sized
+from copy import copy
 from enum import Enum, unique
 from functools import cached_property
 from pathlib import Path
-from typing import NamedTuple, final
+from typing import Any, NamedTuple, final
 
-from torch.utils.data import Dataset
+from torch.utils.data import Dataset, default_collate
 
 from qai_hub_models.utils.asset_loaders import LOCAL_STORE_DEFAULT_PATH
 from qai_hub_models.utils.input_spec import InputSpec
@@ -34,6 +35,30 @@ class DatasetSplit(Enum):
     TEST = 2
 
 
+class AugmentedLabelDataset(Dataset):
+    """
+    Augment labels to a dataset (making the label a tuple, if labels are
+    already present).
+    """
+
+    def __init__(self, base_dataset, extra_data):
+        self.base_dataset = base_dataset
+        self.extra_data = extra_data
+        self.extra_len = len(extra_data)
+
+    def __len__(self):
+        return len(self.base_dataset)
+
+    def __getitem__(self, idx):
+        item = copy(self.base_dataset[idx])
+        extra_item = self.extra_data[idx % self.extra_len]
+        if "label" in item:
+            item["label"] = (item["label"], extra_item)
+        else:
+            item["label"] = extra_item
+        return item
+
+
 class DatasetMetadata(NamedTuple):
     """Metadata about the dataset to publish on the website."""
 
@@ -80,6 +105,13 @@ def __init__(
         self.input_spec = input_spec
         self.download_data()
 
+    @staticmethod
+    def collate_fn(batch: Any) -> Any:
+        """
+        To be passed into DataLoader(..., collate_fn=...).
+        """
+        return default_collate(batch)
+
     @final
     def download_data(self) -> None:
         if self._validate_data():