Fix the pipeline breaks dues to the MSVC 19.40 and numpy 2.0 release (#747)

* dd "-allow-unsupported-compiler" flags to Windows CUDA flags

inspired by this PR: microsoft/onnxruntime#21004

* switch to cmake command line

* handle the issues caused by the latest MSVC release

* correct the typo

* correct the parameter

* try one dash again

* use the installed cmake

* use cmake standalone installation firstly

* use the standalone cmake in win32 python too

* fix it more

* one more try

* fix the MacOS pipeline issue

* fix the pip command line

Author: wenbingl

.pipelines/ci.yml

@@ -126,7 +126,7 @@ stages:
 
     - script: |
         python -m pip install --upgrade setuptools pip
-        python -m pip install numpy
+        python -m pip install 'numpy < 2.0.0'
         export OCOS_NO_OPENCV=1
         export OCOS_SCB_DEBUG=1
         CPU_NUMBER=8 python -m pip install -e .
@@ -322,6 +322,7 @@ stages:
         python -m pip install --upgrade pip
         python -m pip install --upgrade setuptools
         python -m pip install --upgrade wheel
+        python -m pip install 'numpy < 2.0.0'
         python -m pip install onnxruntime==$(ort.version)
       displayName: Install requirements
 
@@ -513,7 +514,7 @@ stages:
 
     - script: |
         python -m pip install --upgrade setuptools pip
-        python -m pip install numpy
+        python -m pip install "numpy < 2.0.0"
         set OCOS_NO_OPENCV=1
         set OCOS_SCB_DEBUG=1
         python -m pip install -v -e .
@@ -570,7 +571,9 @@ stages:
 
       - script: |
           set CUDA_PATH=$(Agent.TempDirectory)\v11.8
-          call .\build.bat -T cuda="%CUDA_PATH%" -DOCOS_ENABLE_CTEST=ON -DOCOS_USE_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=70;86^
+          call .\build.bat -T cuda="%CUDA_PATH%" -DOCOS_ENABLE_CTEST=ON^
+            -DCMAKE_CUDA_FLAGS_INIT=-allow-unsupported-compiler^
+            -DOCOS_USE_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=70;86^
             -DOCOS_ONNXRUNTIME_VERSION="$(ORT_VERSION)" -DONNXRUNTIME_PKG_DIR=.\onnxruntime-win-x64-gpu-$(ORT_VERSION)
         displayName: build the customop library with onnxruntime
 
@@ -590,7 +593,7 @@ stages:
       - script: |
           set CUDA_PATH=$(Agent.TempDirectory)\v11.8
           python -m pip install --upgrade setuptools pip
-          python -m pip install numpy coloredlogs flatbuffers packaging protobuf sympy
+          python -m pip install "numpy < 2.0.0" coloredlogs flatbuffers packaging protobuf sympy
           python -m pip install onnxruntime-gpu==$(ORT_VERSION)
           python -m pip install -v --config-settings "ortx-user-option=use-cuda,cuda_archs=70;86" .
         displayName: Build and install onnxruntime-extensions CUDA package.

.pyproject/cmdclass.py

@@ -226,6 +226,8 @@ def build_cmake(self, extension):
                 if sys.platform == "win32":
                     cuda_path = os.environ.get("CUDA_PATH")
                     cmake_args += [f'-T cuda={cuda_path}']
+                    # TODO: temporarily add a flag for MSVC 19.40
+                    cmake_args += ['-DCMAKE_CUDA_FLAGS_INIT=-allow-unsupported-compiler']
                 f_ver = ext_fullpath.parent / "_version.py"
                 with f_ver.open('a') as _f:
                     _f.writelines(["\n", f"cuda = \"{cuda_ver}\"", "\n"])
@@ -235,7 +237,8 @@ def build_cmake(self, extension):
                 else:
                     smi = _load_nvidia_smi()
                     if not smi:
-                        raise RuntimeError(f"Cannot detect the CUDA archs from your machine, please specify it by yourself.")
+                        raise RuntimeError(
+                            "Cannot detect the CUDA archs from your machine, please specify it manually.")
                     cmake_args += ['-DCMAKE_CUDA_ARCHITECTURES=' + smi]
 
         # CMake lets you override the generator - we need to check this.
@@ -274,7 +277,6 @@ def build_cmake(self, extension):
                 cmake_args += [
                     "-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))]
 
-
         # overwrite the Python module info if the auto-detection doesn't work.
         # export Python3_INCLUDE_DIRS=/opt/python/cp38-cp38
         # export Python3_LIBRARIES=/opt/python/cp38-cp38
@@ -292,14 +294,18 @@ def build_cmake(self, extension):
             '--parallel' + ('' if cpu_number is None else ' ' + cpu_number)
         ]
         cmake_exe = 'cmake'
-        # unlike Linux/macOS, cmake pip package on Windows fails to build some 3rd party dependencies.
-        # so we have to use the cmake installed from Visual Studio.
-        if os.environ.get(VSINSTALLDIR_NAME):
-            cmake_exe = os.environ[VSINSTALLDIR_NAME] + \
-                        'Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\CMake\\bin\\cmake.exe'
-            # Add this cmake directory into PATH to make sure the child-process still find it.
-            os.environ['PATH'] = os.path.dirname(
-                cmake_exe) + os.pathsep + os.environ['PATH']
+        # if sys.platform == "win32":
+        #     # unlike Linux/macOS, cmake pip package on Windows fails to build some 3rd party dependencies.
+        #     # so we have to use the cmake from a standalone installation or the one from Visual Studio.
+        #     standalone_cmake = os.path.join(os.environ.get("ProgramFiles"), "\\CMake\\bin\\cmake.exe")
+        #     if os.path.exists(standalone_cmake):
+        #         cmake_exe = standalone_cmake
+        #     elif os.environ.get(VSINSTALLDIR_NAME):
+        #         cmake_exe = os.environ[VSINSTALLDIR_NAME] + \
+        #                     'Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\CMake\\bin\\cmake.exe'
+        #     # Add this cmake directory into PATH to make sure the child-process still find it.
+        #     os.environ['PATH'] = os.path.dirname(
+        #         cmake_exe) + os.pathsep + os.environ['PATH']
 
         self.spawn([cmake_exe, '-S', str(project_dir),
                     '-B', str(build_temp)] + cmake_args)

CMakeLists.txt

@@ -36,6 +36,11 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0")
   cmake_policy(SET CMP0077 NEW)
 endif()
 
+# Avoid warning of Calling FetchContent_Populate(GSL) is deprecated
+if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.30.0")
+  cmake_policy(CMP0169 OLD)
+endif()
+
 # Needed for Java
 set(CMAKE_C_STANDARD 99)
 

build.bat

@@ -1,12 +1,21 @@
 @ECHO OFF
 SETLOCAL ENABLEDELAYEDEXPANSION
+
+IF NOT EXIST "%ProgramFiles%\CMake\bin\cmake.exe" GOTO :FIND_VS
+set cmake_exe="%ProgramFiles%\CMake\bin\cmake.exe"
+
+:FIND_VS
 IF DEFINED VSINSTALLDIR GOTO :VSDEV_CMD
 set _VSFINDER=%~dp0tools\get_vsdevcmd.ps1
 for /f "tokens=* USEBACKQ" %%i in (
     `powershell -NoProfile -ExecutionPolicy Bypass -File "%_VSFINDER%"`) do call "%%i"
 
 IF NOT DEFINED VSINSTALLDIR GOTO :NOT_FOUND
 
+IF DEFINED cmake_exe GOTO :CMAKE_DEF
+set cmake_exe="%VSINSTALLDIR%Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe"
+
+:CMAKE_DEF
 IF "%1" == "-A" GOTO :VSDEV_CMD
 set GEN_PLATFORM=-A x64
 
@@ -16,8 +25,8 @@ IF "%VisualStudioVersion:~0,2%" == "16" GOTO :START_BUILD
 set GENERATOR="Visual Studio 17 2022"
 
 :START_BUILD
-set cmake_exe="%VSINSTALLDIR%Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe"
 mkdir .\out\Windows\ 2>NUL
+ECHO %cmake_exe% -G %GENERATOR% %GEN_PLATFORM% %* -B out\Windows -S .
 %cmake_exe% -G %GENERATOR% %GEN_PLATFORM% %* -B out\Windows -S .
 IF %ERRORLEVEL% NEQ 0 EXIT /B %ERRORLEVEL%
 %cmake_exe% --build out\Windows --config RelWithDebInfo

cmake/externals/opencv-no-rtti.patch

@@ -39,6 +39,19 @@ index d95e5db163..db185453df 100644
  include(cmake/OpenCVCompilerOptions.cmake)
 
  ocv_cmake_hook(POST_COMPILER_OPTIONS)
+diff --git a/cmake/OpenCVDetectCXXCompiler.cmake b/cmake/OpenCVDetectCXXCompiler.cmake
+index 7f229cde96..92e204a5b9 100644
+--- a/cmake/OpenCVDetectCXXCompiler.cmake
++++ b/cmake/OpenCVDetectCXXCompiler.cmake
+@@ -171,7 +171,7 @@ elseif(MSVC)
+     set(OpenCV_RUNTIME vc15)
+   elseif(MSVC_VERSION MATCHES "^192[0-9]$")
+     set(OpenCV_RUNTIME vc16)
+-  elseif(MSVC_VERSION MATCHES "^193[0-9]$")
++  elseif(MSVC_VERSION MATCHES "^19[34][0-9]$")
+     set(OpenCV_RUNTIME vc17)
+   else()
+     message(WARNING "OpenCV does not recognize MSVC_VERSION \"${MSVC_VERSION}\". Cannot set OpenCV_RUNTIME")
 diff --git a/modules/core/include/opencv2/core/ocl.hpp b/modules/core/include/opencv2/core/ocl.hpp
 index 4503fa00dd..642b0508d0 100644
 --- a/modules/core/include/opencv2/core/ocl.hpp

requirements-dev.txt

@@ -1,5 +1,6 @@
 pytest
-onnx             >= 1.9.0
+numpy            < 2.0.0
+onnx             >=1.9.0
 protobuf         < 4.0.0
 # multiple versions of onnxruntime are supported, but only one can be installed at a time
 onnxruntime      >=1.12.0