Setting up PyTorch plugin "bias_act_plugin"... Failed!

[beeternity]

Traceback (most recent call last):
File "/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/utils/cpp_extension.py", line 1808, in _run_ninja_build
subprocess.run(
File "/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/subprocess.py", line 528, in run
raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['ninja', '-v']' returned non-zero exit status 1.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/dssg/home/zn_lzhx/R3GAN_Experiments/train.py", line 320, in
main() # pylint: disable=no-value-for-parameter
File "/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/click/core.py", line 1161, in call
return self.main(*args, **kwargs)
File "/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/click/core.py", line 1082, in main
rv = self.invoke(ctx)
File "/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/click/core.py", line 1443, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/click/core.py", line 788, in invoke
return __callback(*args, **kwargs)
File "/dssg/home/zn_lzhx/R3GAN_Experiments/train.py", line 315, in main
launch_training(c=c, desc=desc, outdir=opts.outdir, dry_run=opts.dry_run)
File "/dssg/home/zn_lzhx/R3GAN_Experiments/train.py", line 93, in launch_training
subprocess_fn(rank=0, c=c, temp_dir=temp_dir)
File "/dssg/home/zn_lzhx/R3GAN_Experiments/train.py", line 44, in subprocess_fn
training_loop.training_loop(rank=rank, **c)
File "/dssg/home/zn_lzhx/R3GAN_Experiments/training/training_loop.py", line 194, in training_loop
img = misc.print_module_summary(G, [z, c])
File "/dssg/home/zn_lzhx/R3GAN_Experiments/torch_utils/misc.py", line 199, in print_module_summary
outputs = module(*inputs)
File "/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1148, in _call_impl
result = forward_call(*input, **kwargs)
File "/dssg/home/zn_lzhx/R3GAN_Experiments/training/networks.py", line 27, in forward
return self.Model(x, c)
File "/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1148, in _call_impl
result = forward_call(*input, **kwargs)
File "/dssg/home/zn_lzhx/R3GAN_Experiments/R3GAN/Networks.py", line 149, in forward
x = Layer(x)
File "/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1148, in _call_impl
result = forward_call(*input, **kwargs)
File "/dssg/home/zn_lzhx/R3GAN_Experiments/R3GAN/Networks.py", line 111, in forward
x = Layer(x)
File "/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1148, in _call_impl
result = forward_call(*input, **kwargs)
File "/dssg/home/zn_lzhx/R3GAN_Experiments/R3GAN/Networks.py", line 44, in forward
y = self.LinearLayer2(self.NonLinearity1(y))
File "/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1148, in call_impl
result = forward_call(*input, **kwargs)
File "/dssg/home/zn_lzhx/R3GAN_Experiments/R3GAN/FusedOperators.py", line 31, in forward
return bias_act.bias_act(x, self.Bias.to(x.dtype), act=BiasedActivationCUDA.Function, gain=1)
File "/dssg/home/zn_lzhx/R3GAN_Experiments/torch_utils/ops/bias_act.py", line 84, in bias_act
if impl == 'cuda' and x.device.type == 'cuda' and init():
File "/dssg/home/zn_lzhx/R3GAN_Experiments/torch_utils/ops/bias_act.py", line 41, in init
plugin = custom_ops.get_plugin(
File "/dssg/home/zn_lzhx/R3GAN_Experiments/torch_utils/custom_ops.py", line 136, in get_plugin
torch.utils.cpp_extension.load(name=module_name, build_directory=cached_build_dir,
File "/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/utils/cpp_extension.py", line 1202, in load
return jit_compile(
File "/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/utils/cpp_extension.py", line 1425, in jit_compile
write_ninja_file_and_build_library(
File "/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/utils/cpp_extension.py", line 1537, in write_ninja_file_and_build_library
run_ninja_build(
File "/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/utils/cpp_extension.py", line 1824, in run_ninja_build
raise RuntimeError(message) from e
RuntimeError: Error building extension 'bias_act_plugin': [1/3] /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/bin/nvcc -ccbin /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/bin/x86_64-conda-linux-gnu-cc -DTORCH_EXTENSION_NAME=bias_act_plugin -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE="gcc" -DPYBIND11_STDLIB="libstdcpp" -DPYBIND11_BUILD_ABI="cxxabi1011" -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include/torch/csrc/api/include -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include/TH -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include/THC -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/include -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/include/python3.9 -D_GLIBCXX_USE_CXX11_ABI=0 -D__CUDA_NO_HALF_OPERATORS -D__CUDA_NO_HALF_CONVERSIONS -D__CUDA_NO_BFLOAT16_CONVERSIONS -D__CUDA_NO_HALF2_OPERATORS --expt-relaxed-constexpr -gencode=arch=compute_70,code=compute_70 -gencode=arch=compute_70,code=sm_70 --compiler-options '-fPIC' --use_fast_math --allow-unsupported-compiler -std=c++14 -c /dssg/home/zn_lzhx/.cache/torch_extensions/py39_cu113/bias_act_plugin/923aace9669174ee9624e9f9a8189528-tesla-v100-pcie-32gb/bias_act.cu -o bias_act.cuda.o
FAILED: bias_act.cuda.o
/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/bin/nvcc -ccbin /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/bin/x86_64-conda-linux-gnu-cc -DTORCH_EXTENSION_NAME=bias_act_plugin -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE="gcc" -DPYBIND11_STDLIB="libstdcpp" -DPYBIND11_BUILD_ABI="cxxabi1011" -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include/torch/csrc/api/include -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include/TH -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include/THC -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/include -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/include/python3.9 -D_GLIBCXX_USE_CXX11_ABI=0 -D__CUDA_NO_HALF_OPERATORS -D__CUDA_NO_HALF_CONVERSIONS -D__CUDA_NO_BFLOAT16_CONVERSIONS -D__CUDA_NO_HALF2_OPERATORS --expt-relaxed-constexpr -gencode=arch=compute_70,code=compute_70 -gencode=arch=compute_70,code=sm_70 --compiler-options '-fPIC' --use_fast_math --allow-unsupported-compiler -std=c++14 -c /dssg/home/zn_lzhx/.cache/torch_extensions/py39_cu113/bias_act_plugin/923aace9669174ee9624e9f9a8189528-tesla-v100-pcie-32gb/bias_act.cu -o bias_act.cuda.o
x86_64-conda-linux-gnu-cc: fatal error: cannot execute 'cc1plus': execvp: No such file or directory
compilation terminated.
nvcc fatal : Failed to preprocess host compiler properties.
[2/3] c++ -MMD -MF bias_act.o.d -DTORCH_EXTENSION_NAME=bias_act_plugin -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE="_gcc" -DPYBIND11_STDLIB="_libstdcpp" -DPYBIND11_BUILD_ABI="_cxxabi1011" -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include/torch/csrc/api/include -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include/TH -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include/THC -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/include -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/include/python3.9 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++14 -c /dssg/home/zn_lzhx/.cache/torch_extensions/py39_cu113/bias_act_plugin/923aace9669174ee9624e9f9a8189528-tesla-v100-pcie-32gb/bias_act.cpp -o bias_act.o
FAILED: bias_act.o
c++ -MMD -MF bias_act.o.d -DTORCH_EXTENSION_NAME=bias_act_plugin -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE="_gcc" -DPYBIND11_STDLIB="_libstdcpp" -DPYBIND11_BUILD_ABI="_cxxabi1011" -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include/torch/csrc/api/include -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include/TH -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include/THC -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/include -isystem /dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/include/python3.9 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++14 -c /dssg/home/zn_lzhx/.cache/torch_extensions/py39_cu113/bias_act_plugin/923aace9669174ee9624e9f9a8189528-tesla-v100-pcie-32gb/bias_act.cpp -o bias_act.o
In file included from /dssg/home/zn_lzhx/.cache/torch_extensions/py39_cu113/bias_act_plugin/923aace9669174ee9624e9f9a8189528-tesla-v100-pcie-32gb/bias_act.cpp:10:
/dssg/apps/lico/ohpc_pub/apps/anaconda3/envs/R3GAN_env/lib/python3.9/site-packages/torch/include/ATen/cuda/CUDAContext.h:5:10: fatal error: cuda_runtime_api.h: No such file or directory
#include <cuda_runtime_api.h>
^~~~~~~~~~~~~~~~~~~~
compilation terminated.
ninja: build stopped: subcommand failed.

[Skylion007]

This is an environment issue with your PyTorch installation. You need both the CUDA runtime dependencies and compilation dependencies. Usually only the runtime dependencies are installed by default.

You are missing the NVCC compiler and other features in your CUDA environment for compiling PyTorch CUDA extensions on the fly. Please add a conda package for an NVCC compile that matches your cudatoolkit version. There is cuda-nvcc for example in the nvidia conda channel and conda-forge.

You also need the appropriate headers installed.

[beeternity]

Thanks! Here is my environment dependencies:

_libgcc_mutex=0.1=main
_openmp_mutex=5.1=1_gnu
_sysroot_linux-64_curr_repodata_hack=3=haa98f57_10
binutils_impl_linux-64=2.35.1=h27ae35d_9
binutils_linux-64=2.35.1=h454624a_30
blas=1.0=mkl
brotli-python=1.0.9=py39h6a678d5_9
bzip2=1.0.8=h5eee18b_6
ca-certificates=2024.12.31=h06a4308_0
certifi=2025.1.31=py39h06a4308_0
charset-normalizer=3.3.2=pyhd3eb1b0_0
click=8.1.8=pypi_0
contourpy=1.3.0=pypi_0
cuda-nvcc=11.3.58=h2467b9f_0
cudatoolkit=11.3.1=h2bc3f7f_2
cycler=0.12.1=pypi_0
ffmpeg=4.3=hf484d3e_0
fonttools=4.56.0=pypi_0
freetype=2.12.1=h4a9f257_0
gcc_impl_linux-64=9.3.0=h6df7d76_17
gcc_linux-64=9.3.0=h1ee779e_30
gmp=6.3.0=h6a678d5_0
gnutls=3.6.15=he1e5248_0
idna=3.7=py39h06a4308_0
imageio=2.37.0=pypi_0
importlib-resources=6.5.2=pypi_0
intel-openmp=2023.1.0=hdb19cb5_46306
jpeg=9e=h5eee18b_3
kernel-headers_linux-64=3.10.0=h57e8cba_10
kiwisolver=1.4.7=pypi_0
lame=3.100=h7b6447c_0
lcms2=2.16=hb9589c4_0
ld_impl_linux-64=2.35.1=h7274673_9
lerc=4.0.0=h6a678d5_0
libdeflate=1.22=h5eee18b_0
libffi=3.4.4=h6a678d5_1
libgcc-devel_linux-64=9.3.0=hb95220a_17
libgcc-ng=11.2.0=h1234567_1
libgomp=11.2.0=h1234567_1
libiconv=1.16=h5eee18b_3
libidn2=2.3.4=h5eee18b_0
libpng=1.6.39=h5eee18b_0
libstdcxx-ng=11.2.0=h1234567_1
libtasn1=4.19.0=h5eee18b_0
libtiff=4.5.1=hffd6297_1
libunistring=0.9.10=h27cfd23_0
libwebp-base=1.3.2=h5eee18b_1
lz4-c=1.9.4=h6a678d5_1
matplotlib=3.9.4=pypi_0
mkl=2023.1.0=h213fc3f_46344
mkl-service=2.4.0=py39h5eee18b_2
mkl_fft=1.3.11=py39h5eee18b_0
mkl_random=1.2.8=py39h1128e8f_0
ncurses=6.4=h6a678d5_0
nettle=3.7.3=hbbd107a_1
ninja=1.11.1.3=pypi_0
numpy=2.0.1=py39h5f9d8c6_1
numpy-base=2.0.1=py39hb5e798b_1
openh264=2.1.1=h4ff587b_0
openjpeg=2.5.2=he7f1fd0_0
openssl=3.0.15=h5eee18b_0
packaging=24.2=pypi_0
pillow=11.1.0=py39hcea889d_0
pip=25.0=py39h06a4308_0
psutil=7.0.0=pypi_0
pyparsing=3.2.1=pypi_0
pysocks=1.7.1=py39h06a4308_0
python=3.9.21=he870216_1
python-dateutil=2.9.0.post0=pypi_0
pytorch=1.12.1=py3.9_cuda11.3_cudnn8.3.2_0
pytorch-mutex=1.0=cuda
readline=8.2=h5eee18b_0
requests=2.32.3=py39h06a4308_1
scipy=1.13.1=pypi_0
setuptools=75.8.0=py39h06a4308_0
six=1.17.0=pypi_0
sqlite=3.45.3=h5eee18b_0
sysroot_linux-64=2.17=h57e8cba_10
tbb=2021.8.0=hdb19cb5_0
tk=8.6.14=h39e8969_0
torchaudio=0.12.1=py39_cu113
torchvision=0.13.1=py39_cu113
tqdm=4.67.1=pypi_0
typing_extensions=4.12.2=py39h06a4308_0
tzdata=2025a=h04d1e81_0
urllib3=2.3.0=py39h06a4308_0
wheel=0.45.1=py39h06a4308_0
xz=5.6.4=h5eee18b_1
zipp=3.21.0=pypi_0
zlib=1.2.13=h5eee18b_1
zstd=1.5.6=hc292b87_0

[beeternity]

nvcc --version

nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2021 NVIDIA Corporation
Built on Sun_Mar_21_19:15:46_PDT_2021
Cuda compilation tools, release 11.3, V11.3.58
Build cuda_11.3.r11.3/compiler.29745058_0

gcc version 9.3.0

[Skylion007]

That version of CUDA and PyTorch is very old. I am not sure they are even fully supported by our codebase.