support for paddle (#16)

support  for paddle

support gloo in cpu case

try nccl in gpu case

update test case

fix the bug in nccl

support gds

fix merge bug

add paddle unittest

confuse about test_memmove

fix uint16

add the paddlepaddle-gpu to project.toml

paddlepaddle set 3.0 version

paddlepaddle-gpu==3.0.0 should install from www.paddlepaddle.org.cn

remove cpp.cpythonxxx.so

add paddle_loaded

Signed-off-by: Takeshi Yoshimura <tyos@jp.ibm.com>
Signed-off-by: zeroRains <linjunlu@zerorains.top>

Author: zeroRains

.github/workflows/test-paddle.yaml

@@ -55,4 +55,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: pytest-log-paddle-${{ matrix.python-version }}
-          path: /tmp/pytest-log
+          path: /tmp/pytest-log

.gitignore

@@ -5,4 +5,8 @@ dist/
 htmlcov/
 .coverage
 .coverage_*
-.pytest_cache/
+.pytest_cache/
+.vscode
+*.log
+*.pyc
+examples/paddle_case/log

examples/paddle_case/a_paddle.safetensors

@@ -0,0 +1,6 @@
+import os
+import paddle
+t0 = paddle.concat([paddle.full((1,8), i, dtype=paddle.float16) for i in range(0, 16)], dim=0)
+from safetensors.paddle import save_file
+for file_prefix in ["a", "b"]:
+    save_file({f"{file_prefix}0": t0}, f"{file_prefix}_paddle.safetensors", metadata={"fst": "sample"})

examples/paddle_case/b_paddle.safetensors

@@ -0,0 +1,32 @@
+# !/usr/bin/env python3
+# PIDS=()
+
+# runner="python -m paddle.distributed.launch"
+
+# cd paddle_case
+# ${runner} --nnodes=2 --master=127.0.0.1:12345 --rank=0 run_parallel.py &
+# PIDS+=($!)
+# ${runner} --nnodes=2 --master=127.0.0.1:12345 --rank=1 run_parallel.py &
+# PIDS+=($!)
+# wait "${PIDS[@]}"
+
+import paddle
+import paddle.distributed as dist
+from fastsafetensors import SafeTensorsFileLoader
+dist.init_parallel_env()
+backend = "nccl" if paddle.is_compiled_with_cuda() else "gloo"
+pg = dist.new_group(ranks=[0,1], backend=backend)
+device = "gpu:0" if paddle.is_compiled_with_cuda() else "cpu"
+loader = SafeTensorsFileLoader(pg, device, nogds=False, debug_log=True, framework="paddle")
+loader.add_filenames({0: ["a_paddle.safetensors"], 1:["b_paddle.safetensors"]}) # {rank: files}
+
+# load a.safetensors to rank 0 GPU and b.safetensors to rank 1 GPU
+fb = loader.copy_files_to_device()
+
+# every rank must call get_tensor and get_sharded in the same order since they internally call paddle.distributed collective ops
+tensor_a0 = fb.get_tensor(tensor_name="a0") # broadcast
+tensor_b0_sharded = fb.get_sharded(tensor_name="b0", dim=1) # partition and scatter
+print(f"RANK {pg.process_group.rank()}: tensor_a0={tensor_a0}")
+print(f"RANK {pg.process_group.rank()}: tensor_b0_sharded={tensor_b0_sharded}")
+fb.close()
+loader.close()

examples/paddle_case/gen.py

@@ -0,0 +1,13 @@
+import paddle
+from fastsafetensors import SafeTensorsFileLoader, SingleGroup
+device = "gpu:0" if paddle.is_compiled_with_cuda() else "cpu"
+loader = SafeTensorsFileLoader(SingleGroup(), device, nogds=False, debug_log=True, framework="paddle")
+loader.add_filenames({0: ["a_paddle.safetensors", "b_paddle.safetensors"]}) # {rank: files}
+fb = loader.copy_files_to_device()
+tensor_a0 = fb.get_tensor(tensor_name="a0")
+tensor_b0 = fb.get_tensor(tensor_name="b0")
+print(f"a0: {tensor_a0}")
+mycat = paddle.concat([tensor_a0, tensor_b0], axis=1)
+print(f"cat: {mycat}, size={mycat.size}")
+fb.close()
+loader.close()

examples/paddle_case/run_parallel.py

@@ -0,0 +1,15 @@
+# !/usr/bin/env python3
+PIDS=()
+
+runner="python -m paddle.distributed.launch"
+# runner="torchrun"
+
+cd paddle_case
+rm -rf log
+# one machine multy gpu (case : 1 machine 2 gpus)
+# different to torch script because the paddle distributed use nccl to communicate in gpus
+CUDA_VISIBLE_DEVICES=0 ${runner} --nnodes=2 --master=127.0.0.1:8800 --rank=0 run_parallel.py &
+PIDS+=($!)
+CUDA_VISIBLE_DEVICES=1 ${runner} --nnodes=2 --master=127.0.0.1:8800 --rank=1 run_parallel.py &
+PIDS+=($!)
+wait "${PIDS[@]}"

examples/paddle_case/run_single.py

@@ -14,7 +14,7 @@
 dist.barrier()
 pg = dist.group.WORLD
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
-loader = SafeTensorsFileLoader(pg, device, nogds=True, debug_log=True)
+loader = SafeTensorsFileLoader(pg, device, nogds=False, debug_log=True)
 loader.add_filenames({0: ["a.safetensors"], 1:["b.safetensors"]}) # {rank: files}
 
 # load a.safetensors to rank 0 GPU and b.safetensors to rank 1 GPU

examples/run_paddle_parrallel.sh

@@ -5,7 +5,7 @@
 sys.path.insert(0, "/nvme/manish/repos/fastsafetensors/fastsafetensors")
 
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
-loader = SafeTensorsFileLoader(SingleGroup(), device)#, nogds=True, debug_log=True)
+loader = SafeTensorsFileLoader(SingleGroup(), device, nogds=True, debug_log=True)
 
 loader.add_filenames({0: ["a.safetensors"]}) # {rank: files}
 fb = loader.copy_files_to_device()