[Cherry Pick] Remove cuda dependencies from non cuda buids pytorch#152333 (pytorch#153089)

* Remove cuda dependencies from non cuda buids

* regenerate

* regenerate

Author: atalman

.github/scripts/generate_binary_build_matrix.py

@@ -417,8 +417,6 @@ def generate_wheels_matrix(
                         "pytorch_extra_install_requirements": (
                             PYTORCH_EXTRA_INSTALL_REQUIREMENTS["xpu"]
                             if gpu_arch_type == "xpu"
-                            else PYTORCH_EXTRA_INSTALL_REQUIREMENTS[CUDA_STABLE]
-                            if os != "linux"
                             else ""
                         ),
                     }