[huggingface tracer] Add suite based off of results from tracer

BackendBench/huggingface_tracer/README.md

@@ -0,0 +1,87 @@
+# Sample Inputs Schema
+
+This directory contains outputs of the huggingface tracer  which store traced PyTorch operation inputs from HuggingFace models.
+
+'[hf_op_trace.json](https://huggingface.co/datasets/GPUMODE/huggingface_op_trace/resolve/main/hf_op_trace.json)' contains an example of what these look like with the outputs from 20 models.
+
+## Schema Structure
+
+```json
+{
+  "operation_name": {
+    "total_calls": <int>,
+    "unique_input_count": <int>,
+    "unique_inputs": [
+      {
+        "op_name": "<operation_name>",
+        "input_shapes": [<shape_or_null>, ...],
+        "input_dtypes": ["<dtype_string>", ...],
+        "non_tensor_inputs": [<value_or_null_or_tensor_list_ref>, ...],
+        "tensor_lists": {<tensor_list_metadata>},
+        "count": <int>
+      }
+    ]
+  }
+}
+```
+
+## Field Descriptions
+
+- **`input_shapes`**: List of tensor shapes (e.g., `[1, 3, 224, 224]`) or `null` for non-tensor inputs
+- **`input_dtypes`**: List of type strings (e.g., `"torch.float32"`, `"<class 'int'>"`, `"<class 'list'>"`
+- **`non_tensor_inputs`**: Actual non-tensor values, `null` for tensors, or `{"tensor_list_ref": <id>}` for tensor lists
+- **`tensor_lists`**: Metadata for tensor lists, keyed by string IDs:
+  ```json
+  {
+    "0": {
+      "length": <int>,
+      "shapes": [[<shape>], ...],
+      "dtypes": ["<dtype>", ...]
+    }
+  }
+  ```
+- **`count`**: Frequency of this input combination in the traced data
+
+**Note**: All dtypes (in input_dtypes and tensor_lists) are strings, not Python types (e.g., `torch.float32` instead of `float32`) as they are serialized in the JSON file. They should be converted to Python types before use.
+
+## Examples
+
+**Simple tensor input:**
+```json
+"input_shapes": [[2, 13]],
+"input_dtypes": ["torch.int64"],
+"non_tensor_inputs": [null]
+```
+
+**Tensor list input:**
+```json
+"input_shapes": [null, null],
+"input_dtypes": ["<class 'list'>", "<class 'int'>"],
+"non_tensor_inputs": [{"tensor_list_ref": 0}, 1],
+"tensor_lists": {
+  "0": {
+    "length": 3,
+    "shapes": [[1, 128, 20, 20], [1, 128, 20, 20], [1, 128, 20, 20]],
+    "dtypes": ["torch.float32", "torch.float32", "torch.float32"]
+  }
+}
+```
+
+**Example entry with non-tensor inputs**
+```json
+"convolution.default": {
+  "total_calls": 108,
+  "unique_input_count": 67,
+  "unique_inputs": [
+    {
+      "op_name": "convolution.default",
+      "input_shapes": [[1, 256, 14, 14], [1024, 256, 1, 1], null, null, null, null, null, null, null],
+      "input_dtypes": ["torch.float32", "torch.float32", "<class 'NoneType'>", "<class 'list'>", "<class 'list'>", "<class 'list'>", "<class 'bool'>", "<class 'list'>", "<class 'int'>"],
+      "non_tensor_inputs": [null, null, null, [1, 1], [0, 0], [1, 1], false, [0, 0], 1],
+      "tensor_lists": {},
+      "count": 6
+    },
+    ...
+  ]
+  }
+```

BackendBench/huggingface_tracer/__init__.py

@@ -0,0 +1,20 @@
+"""
+HuggingFace Tracer Test Suite Package.
+
+This package provides functionality for creating and running test suites
+based on HuggingFace tracer data.
+"""
+
+from .suite import (
+    build_huggingface_tracer_tests,
+    HuggingFaceTracerOpTest,
+    HuggingFaceTracerTest,
+    HuggingFaceTracerTestSuite,
+)
+
+__all__ = [
+    "HuggingFaceTracerTest",
+    "HuggingFaceTracerOpTest",
+    "HuggingFaceTracerTestSuite",
+    "build_huggingface_tracer_tests",
+]

BackendBench/huggingface_tracer/manual_ops_mapping.json

@@ -0,0 +1,74 @@
+{
+    "lift_fresh": {
+        "cpu": ["torch.float32", "torch.int64", "torch.uint8"],
+        "cuda": ["torch.float32", "torch.int64", "torch.uint8"]
+    },
+    "_to_copy": {
+        "cpu": ["torch.bool", "torch.float32", "torch.int64", "torch.uint8"],
+        "cuda": ["torch.bool", "torch.float32", "torch.int64", "torch.uint8"]
+    },
+    "convolution": {
+        "cpu": ["torch.float32"],
+        "cuda": ["torch.float32"]
+    },
+    "_scaled_dot_product_efficient_attention": {
+        "cpu": ["torch.float32"],
+        "cuda": ["torch.float32"]
+    },
+    "detach": {
+        "cpu": ["torch.float32"],
+        "cuda": ["torch.float32"]
+    },
+    "_has_compatible_shallow_copy_type": {
+        "cpu": ["torch.float32"],
+        "cuda": ["torch.float32"]
+    },
+    "detach_": {
+        "cpu": ["torch.int64"],
+        "cuda": ["torch.int64"]
+    },
+    "is_nonzero": {
+        "cpu": ["torch.bool"],
+        "cuda": ["torch.bool"]
+    },
+    "linalg_vector_norm": {
+        "cpu": ["torch.float32"],
+        "cuda": ["torch.float32"]
+    },
+    "_local_scalar_dense": {
+        "cpu": ["torch.bool", "torch.float32"],
+        "cuda": ["torch.bool", "torch.float32"]
+    },
+    "cudnn_batch_norm": {
+        "cpu": ["torch.float32"],
+        "cuda": ["torch.float32"]
+    },
+    "max_pool2d_with_indices": {
+        "cpu": ["torch.float32"],
+        "cuda": ["torch.float32"]
+    },
+    "copy_": {
+        "cpu": ["torch.float32"],
+        "cuda": ["torch.float32"]
+    },
+    "upsample_nearest2d": {
+        "cpu": ["torch.float32"],
+        "cuda": ["torch.float32"]
+    },
+    "alias": {
+        "cpu": ["torch.float32"],
+        "cuda": ["torch.float32"]
+    },
+    "type_as": {
+        "cpu": ["torch.int32", "torch.int64"],
+        "cuda": ["torch.int32", "torch.int64"]
+    },
+    "_scaled_dot_product_flash_attention_for_cpu": {
+        "cpu": ["torch.float32"],
+        "cuda": []
+    },
+    "_softmax": {
+        "cpu": ["torch.float32"],
+        "cuda": ["torch.float32"]
+    }
+}