Skip to content

add bnb support for Ascend NPU #31512

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Dec 23, 2024
Merged

add bnb support for Ascend NPU #31512

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c9f2047
add bnb support for Ascend NPU
ji-huazhong Jun 17, 2024
a411f18
delete comment
ji-huazhong Dec 9, 2024
125f894
Merge branch 'main' into bnb
ji-huazhong Dec 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion src/transformers/quantizers/quantizer_bnb_4bit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
is_accelerate_available,
is_bitsandbytes_available,
is_torch_available,
is_torch_npu_available,
is_torch_xpu_available,
logging,
)
Expand Down Expand Up @@ -171,6 +172,9 @@ def create_quantized_param(

old_value = getattr(module, tensor_name)

# `torch.Tensor.to(<int num>)` is not supported by `torch_npu` (see this [issue](https://github.com/Ascend/pytorch/issues/16)).
if isinstance(target_device, int) and is_torch_npu_available():
target_device = f"npu:{target_device}"
if tensor_name == "bias":
if param_value is None:
new_value = old_value.to(target_device)
Expand Down Expand Up @@ -259,11 +263,12 @@ def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":
torch_dtype = torch.float16
return torch_dtype

# Copied from transformers.quantizers.quantizer_bnb_8bit.Bnb8BitHfQuantizer.update_device_map
def update_device_map(self, device_map):
if device_map is None:
if torch.cuda.is_available():
device_map = {"": torch.cuda.current_device()}
elif is_torch_npu_available():
device_map = {"": f"npu:{torch.npu.current_device()}"}
elif is_torch_xpu_available():
device_map = {"": f"xpu:{torch.xpu.current_device()}"}
else:
Expand Down