Revert "Cherry Pick of M1 build pytorch#6847" (pytorch#6950)

This was reverted on main

Author: atalman

.github/workflows/build_wheels_macos.yml

@@ -78,7 +78,7 @@ on:
         description: Set to true if setup-miniconda is needed
         required: false
         type: boolean
-        default: true
+        default: false
       delocate-wheel:
         description: "Whether to run delocate-wheel after building."
         required: false