Skip to content

Commit 916da6a

Browse files
[OpenVINO] Fix jais-13b int4 config (#1521)
* Add config only if nncf is installed * Fix
1 parent 3a5fc48 commit 916da6a

File tree

1 file changed

+16
-9
lines changed

1 file changed

+16
-9
lines changed

optimum/intel/openvino/configuration.py

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -350,17 +350,24 @@ class OVQuantizationMethod(str, Enum):
350350
"sym": False,
351351
"group_size": -1,
352352
},
353-
"inceptionai/jais-13b": {
354-
"bits": 4,
355-
"sym": False,
356-
"group_size": 128,
357-
"ratio": 1.0,
358-
"advanced_parameters": nncf.AdvancedCompressionParameters(
359-
group_size_fallback_mode=nncf.GroupSizeFallbackMode.ADJUST,
360-
),
361-
},
362353
}
363354

355+
if is_nncf_available():
356+
# TODO: Remove after update to NNCF 2.19 because `group_size_fallback` argument will be added to OVWeightQuantizationConfig
357+
_DEFAULT_4BIT_WQ_CONFIGS.update(
358+
{
359+
"inceptionai/jais-13b": {
360+
"bits": 4,
361+
"sym": False,
362+
"group_size": 128,
363+
"ratio": 1.0,
364+
"advanced_parameters": nncf.AdvancedCompressionParameters(
365+
group_size_fallback_mode=nncf.GroupSizeFallbackMode.ADJUST,
366+
),
367+
}
368+
}
369+
)
370+
364371
# Add configs for model id aliases
365372
# The list below contains pairs of model ids: config for the second model id will be copied from the first model id.
366373
model_id_aliases = [

0 commit comments

Comments
 (0)