Fix MSVC MT/MD incompatibility in PYBIND11_BUILD_ABI (#4953)

* Fix MSVC MT/MD incompatibility in PYBIND11_BUILD_ABI

* Update comment about which PR

* Use msvc major version

* Use _MSC_VER/100

* Fix figuring out MD vs MT

* Add some test runs

* Skip one test

* Fix preprocessor

* simplify code

* fix if

* support only msvc 19

* Fold in changes from experimental PR #5411. Polish error messages.

* Remove `&& defined(_DLL)` (TBD: is it needed? but what is correct?)

* Fix MT vs MD

* Add a couple comments, based on #4953 (comment) (posted by @isuruf).

* Replace misleading comment: NVHPC is NOT outdated.

* Update include/pybind11/detail/internals.h

Co-authored-by: Robert Maynard <[email protected]>

---------

Co-authored-by: Ralf W. Grosse-Kunstleve <[email protected]>
Co-authored-by: Ralf W. Grosse-Kunstleve <[email protected]>
Co-authored-by: Robert Maynard <[email protected]>

Author: 3 people

.github/workflows/ci.yml

@@ -65,6 +65,24 @@ jobs:
           # Inject a couple Windows 2019 runs
           - runs-on: windows-2019
             python: '3.9'
+          # Inject a few runs with different runtime libraries
+          - runs-on: windows-2022
+            python: '3.9'
+            args: >
+              -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded
+          - runs-on: windows-2022
+            python: '3.10'
+            args: >
+              -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDLL
+          # This needs a python built with MTd
+          # - runs-on: windows-2022
+          #   python: '3.11'
+          #   args: >
+          #     -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebug
+          - runs-on: windows-2022
+            python: '3.12'
+            args: >
+              -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebugDLL
           # Extra ubuntu latest job
           - runs-on: ubuntu-latest
             python: '3.11'

include/pybind11/detail/internals.h

@@ -310,15 +310,29 @@ struct type_info {
 #    endif
 #endif
 
-/// On Linux/OSX, changes in __GXX_ABI_VERSION__ indicate ABI incompatibility.
-/// On MSVC, changes in _MSC_VER may indicate ABI incompatibility (#2898).
 #ifndef PYBIND11_BUILD_ABI
-#    if defined(__GXX_ABI_VERSION)
+#    if defined(__GXX_ABI_VERSION) // Linux/OSX.
 #        define PYBIND11_BUILD_ABI "_cxxabi" PYBIND11_TOSTRING(__GXX_ABI_VERSION)
-#    elif defined(_MSC_VER)
-#        define PYBIND11_BUILD_ABI "_mscver" PYBIND11_TOSTRING(_MSC_VER)
+#    elif defined(_MSC_VER)               // See PR #4953.
+#        if defined(_MT) && defined(_DLL) // Corresponding to CL command line options /MD or /MDd.
+#            if (_MSC_VER) / 100 == 19
+#                define PYBIND11_BUILD_ABI "_md_mscver19"
+#            else
+#                error "Unknown major version for MSC_VER: PLEASE REVISE THIS CODE."
+#            endif
+#        elif defined(_MT) // Corresponding to CL command line options /MT or /MTd.
+#            define PYBIND11_BUILD_ABI "_mt_mscver" PYBIND11_TOSTRING(_MSC_VER)
+#        else
+#            if (_MSC_VER) / 100 == 19
+#                define PYBIND11_BUILD_ABI "_none_mscver19"
+#            else
+#                error "Unknown major version for MSC_VER: PLEASE REVISE THIS CODE."
+#            endif
+#        endif
+#    elif defined(__NVCOMPILER)       // NVHPC (PGI-based).
+#        define PYBIND11_BUILD_ABI "" // TODO: What should be here, to prevent UB?
 #    else
-#        define PYBIND11_BUILD_ABI ""
+#        error "Unknown platform or compiler: PLEASE REVISE THIS CODE."
 #    endif
 #endif