convert : use writeable buffer for remote lazy tensors

Author: compilade

convert_hf_to_gguf.py

@@ -5416,7 +5416,8 @@ def from_remote_tensor(cls, remote_tensor: gguf.utility.RemoteTensor):
         dtype = cls._dtype_str_map[remote_tensor.dtype]
         shape = remote_tensor.shape
         meta = cls.meta_with_dtype_and_shape(dtype, shape)
-        lazy = cls(meta=meta, args=(remote_tensor,), func=lambda r: torch.frombuffer(r.data(), dtype=dtype).reshape(shape))
+        func = lambda r: torch.frombuffer(r.data(), dtype=dtype).reshape(shape)
+        lazy = cls(meta=meta, args=(remote_tensor,), func=func)
         return cast(torch.Tensor, lazy)
 
     @classmethod

gguf-py/gguf/utility.py

@@ -81,9 +81,10 @@ class RemoteTensor:
     size: int
     url: str
 
-    def data(self) -> bytes:
+    def data(self) -> bytearray:
         # TODO: handle request errors (maybe with limited retries?)
-        data = SafetensorRemote.get_data_by_range(url=self.url, start=self.offset_start, size=self.size)
+        # NOTE: using a bytearray, otherwise PyTorch complains the buffer is not writeable
+        data = bytearray(SafetensorRemote.get_data_by_range(url=self.url, start=self.offset_start, size=self.size))
         return data