SG-41413 - CI - Dependencies to build python packages from source (#1014)

###

### Linked issues
n/a

### Summarize your change.
Install and set-up dependencies for Python/Pip to build its dependencies
from source.

### Describe the reason for the change.
Follow up on
#1013 to install
the needed dependencies to build Numpy and Cryptography from source.

### Describe what you have tested and on which operating system.
CI

### Add a list of changes, and note any that might need special
attention during the review.

### If possible, provide screenshots.

---------

Signed-off-by: Roger Nelson <[email protected]>
Signed-off-by: Cédrik Fuoco <[email protected]>
Co-authored-by: Roger Nelson <[email protected]>

Author: cedrik-fuoco-adsk

.github/workflows/ci.yml

@@ -246,6 +246,43 @@ jobs:
             exit 1
           fi
 
+      - name: Install GCC 11 toolchain for Rocky Linux 8
+        if: ${{ matrix.rocky-version == '8' }}
+        run: |
+          retry_count=0
+          max_retries=5
+          
+          until [ $retry_count -ge $max_retries ]; do
+            echo "Attempt $((retry_count + 1)) of $max_retries"
+            
+            if dnf install -y gcc-toolset-11-toolchain; then
+              echo "GCC 11 toolchain installed successfully"
+              break
+            else
+              retry_count=$((retry_count + 1))
+              if [ $retry_count -lt $max_retries ]; then
+                echo "Attempt $retry_count failed. Retrying in 5 seconds..."
+                sleep 5
+              fi
+            fi
+          done
+          
+          if [ $retry_count -ge $max_retries ]; then
+            echo "Failed to install GCC 11 toolchain after $max_retries attempts"
+            exit 1
+          fi
+
+      - name: Enable GCC 11 toolchain for Rocky Linux 8
+        if: ${{ matrix.rocky-version == '8' }}
+        run: |
+          source /opt/rh/gcc-toolset-11/enable
+          echo "/opt/rh/gcc-toolset-11/root/usr/bin" >> $GITHUB_PATH
+          echo "CC=/opt/rh/gcc-toolset-11/root/usr/bin/gcc" >> $GITHUB_ENV
+          echo "CXX=/opt/rh/gcc-toolset-11/root/usr/bin/g++" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV
+          gcc --version
+          g++ --version
+
       - name: Check out repository code
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # 4.1.7
         with:
@@ -735,6 +772,11 @@ jobs:
         run: |
           python -c "import sys; print(sys.executable)"
 
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@0b1efabc08b657293548b77fb76cc02d26091c7e
+        with:
+          toolchain: stable
+
       - name: Cache CMake for Windows
         uses: actions/cache@v4
         id: windows-cmake-cache
@@ -757,13 +799,20 @@ jobs:
           echo "export ACLOCAL_PATH=/c/msys64/usr/share/aclocal" >> ~/.bash_profile
         shell: msys2 {0}
 
+      - name: Add Rust to PATH for msys2
+        run: |
+          echo "export PATH=\"/c/Users/runneradmin/.cargo/bin:\$PATH\"" >> ~/.bash_profile
+        shell: msys2 {0}
+
       - name: Display environment variables
         run: |
           echo $PATH
           echo $ACLOCAL_PATH
           which python
           which python3
           which cmake
+          rustc --version || echo "WARNING: Rust not found in msys2 PATH"
+          cargo --version || echo "WARNING: Cargo not found in msys2 PATH"
         shell: msys2 {0}
 
       - name: Prepare Qt folder

cmake/dependencies/python3.cmake

@@ -29,19 +29,14 @@ SET(RV_DEPS_PYTHON_VERSION_SHORT
     "${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}"
 )
 
-# This version is used for generating src/build/requirements.txt from requirements.txt.in template All platforms install OpenTimelineIO from git to ensure
-# consistent source builds.
+# This version is used for generating src/build/requirements.txt from requirements.txt.in template
 SET(_opentimelineio_version
     "${RV_DEPS_OTIO_VERSION}"
 )
 
 SET(_pyside_version
     "${RV_DEPS_PYSIDE_VERSION}"
 )
-# Construct the full git URL for pip to use in requirements.txt Using this avoids @ symbol conflicts in CONFIGURE_FILE
-SET(_opentimelineio_pip_url
-    "git+https://github.com/AcademySoftwareFoundation/OpenTimelineIO@v${_opentimelineio_version}#egg=OpenTimelineIO"
-)
 
 SET(_python3_download_url
     "https://github.com/python/cpython/archive/refs/tags/v${_python3_version}.zip"
@@ -77,9 +72,6 @@ SET(_build_dir
     ${RV_DEPS_BASE_DIR}/${_python3_target}/build
 )
 
-# Note: OpenTimelineIO is now installed via requirements.txt from git URL for all platforms. This ensures consistent source builds across Windows, Mac, and
-# Linux.
-
 FETCHCONTENT_DECLARE(
   ${_pyside_target}
   URL ${_pyside_archive_url}
@@ -276,12 +268,21 @@ ELSE()
   SET(_otio_debug_env "")
 ENDIF()
 
-# Single unified command for all platforms and build types
+# Using --no-binary :all: to ensure all packages with native extensions are built from source
+# against our custom Python build, preventing ABI compatibility issues.
 SET(_requirements_install_command
     ${CMAKE_COMMAND} -E env
     ${_otio_debug_env}
+)
+
+# Only set OPENSSL_DIR if we built OpenSSL ourselves (not for Rocky Linux 8 CY2023 which uses system OpenSSL)
+IF(DEFINED RV_DEPS_OPENSSL_INSTALL_DIR)
+  LIST(APPEND _requirements_install_command "OPENSSL_DIR=${RV_DEPS_OPENSSL_INSTALL_DIR}")
+ENDIF()
+
+LIST(APPEND _requirements_install_command
     "CMAKE_ARGS=-DPYTHON_LIBRARY=${_python3_cmake_library} -DPYTHON_INCLUDE_DIR=${_include_dir} -DPYTHON_EXECUTABLE=${_python3_executable}"
-    "${_python3_executable}" -s -E -I -m pip install --upgrade --no-cache-dir --force-reinstall -r "${_requirements_output_file}"
+    "${_python3_executable}" -s -E -I -m pip install --upgrade --no-cache-dir --force-reinstall --no-binary :all: -r "${_requirements_output_file}"
 )
 
 IF(RV_TARGET_WINDOWS)
@@ -383,13 +384,16 @@ IF(RV_TARGET_WINDOWS
    AND CMAKE_BUILD_TYPE MATCHES "^Debug$"
 )
   # OCIO v2.2's pybind11 doesn't find python<ver>.lib in Debug since the name is python<ver>_d.lib.
+  # Also, Rust libraries (like cryptography via pyo3) look for python3.lib.
   ADD_CUSTOM_COMMAND(
     TARGET ${_python3_target}
     POST_BUILD
     COMMENT "Copying Debug Python lib as a unversionned file for Debug"
     COMMAND cmake -E copy_if_different ${_python3_implib} ${_python_release_libpath}
-    COMMAND cmake -E copy_if_different ${_python3_implib} ${_python_release_in_bin_libpath} DEPENDS ${_python3_target} ${_requirements_output_file}
-            ${_requirements_input_file}
+    COMMAND cmake -E copy_if_different ${_python3_implib} ${_python_release_in_bin_libpath}
+    COMMAND cmake -E copy_if_different ${_lib_dir}/python${PYTHON_VERSION_MAJOR}_d.lib ${_lib_dir}/python${PYTHON_VERSION_MAJOR}.lib
+    COMMAND cmake -E copy_if_different ${_bin_dir}/python${PYTHON_VERSION_MAJOR}_d.lib ${_bin_dir}/python${PYTHON_VERSION_MAJOR}.lib
+    DEPENDS ${_python3_target} ${_requirements_output_file} ${_requirements_input_file}
   )
 ENDIF()
 

docs/build_system/config_linux_rocky89.md

@@ -73,8 +73,9 @@ All other dependencies are shared across variations.
 1. [Install tools and build dependencies](rocky_install_tools_and_dependencies)
 2. [Install Pyenv / Python](rocky_install_pyenv_and_python)
 3. [Install CMake](rocky_install_cmake)
-4. [Install Qt](rocky_install_qt)
-5. [Build Open RV](rocky_build_openrv)
+4. [Install Rust](rocky_install_rust)
+5. [Install Qt](rocky_install_qt)
+6. [Build Open RV](rocky_build_openrv)
 6. [Building with Docker (Optional)](rocky_building_with_docker)
 
 
@@ -164,8 +165,30 @@ cmake --version  # confirm the version of your newly installed version of CMake
 cmake version 3.31.6
 ```
 
+(rocky_install_rust)=
+### 4. Install Rust
+
+````{warning}
+Rust version **1.92 or later** is required to build certain Python dependencies (such as cryptography).
+````
+
+Install Rust using rustup, and add its installation path to your ~/.bashrc and apply the changes to your current terminal window:
+
+```bash
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> ~/.bashrc
+source ~/.bashrc
+```
+
+Verify that Rust is installed correctly:
+
+```bash
+rustc --version
+cargo --version
+```
+
 (rocky_install_qt)=
-### 4. Install Qt
+### 5. Install Qt
 
 Download the latest open-source [Qt installer](https://www.qt.io/download-open-source). We do not recommend that you install Qt from other installable sources as it may introduce build issues.
 
@@ -193,19 +216,19 @@ Note 1: If you install Qt at a different installation path, you will need to man
 Note 2: Qt modules for Logs, Android, iOS and WebAssembly are not required to build Open RV. 
 
 (rocky_build_openrv)=
-### 5. Build Open RV
+### 6. Build Open RV
 
 Once the platform-specific installation process is complete, building Open RV follows the same process for all platforms. Please refer to the [Common Build Instructions](config_common_build.md) for the complete build process.
 
 (rocky_building_with_docker)=
-### 6. Building with Docker (Optional)
+### 7. Building with Docker (Optional)
 
 To build Open RV using Docker, use the provided Dockerfile found in this repository, which should already contain all required dependencies. 
 
 Please go through the cloning procedure found in the [common build process](config_common_build.md). Once cloned, get back here to build the docker image, run the container, and build Open RV within the docker container.
 
 
-#### 5.1. Build the image and run
+#### 7.1. Build the image and run
 
 ````{tabs}
 ```{code-tab} bash Rocky 9
@@ -223,7 +246,7 @@ docker run -d openrv-rocky8 /bin/bash -c "sleep infinity"
 
 
 
-#### 5.2. Create and run the container
+#### 7.2. Create and run the container
 ```bash
 # Lookup the container id for openrv-rocky{8/9}
 docker container ls
@@ -237,11 +260,11 @@ docker container ls
 docker container exec -it <id> /bin/bash  # replace 'id' with your value
 ```
 
-#### 5.3. Build Open RV in the container
+#### 7.3. Build Open RV in the container
 
 Once you are into the container, you can follow the [common build process](config_common_build.md) to build Open RV.
 
-#### 5.4. Copy the stage folder outside of the container
+#### 7.4. Copy the stage folder outside of the container
 
 If you are on a host that is the same as, or compatible with, your version of Rocky Linux, you can copy the stage folder outside of the container and execute Open RV.
 

docs/build_system/config_macos.md

@@ -126,13 +126,18 @@ Most of the build requirements can be installed by running the following brew in
 
 ````{tabs}
 ```{code-tab} bash VFX-CY2024
-brew install ninja readline sqlite3 xz zlib tcl-tk@8 autoconf automake libtool [email protected] yasm clang-format black meson nasm pkg-config glew
+brew install ninja readline sqlite3 xz zlib tcl-tk@8 autoconf automake libtool [email protected] yasm clang-format black meson nasm pkg-config glew rust
 ```
 ```{code-tab} bash VFX-CY2023
-brew install ninja readline sqlite3 xz zlib tcl-tk@8 autoconf automake libtool [email protected] yasm clang-format black meson nasm pkg-config glew
+brew install ninja readline sqlite3 xz zlib tcl-tk@8 autoconf automake libtool [email protected] yasm clang-format black meson nasm pkg-config glew rust
 ```
 ````
 
+````{warning}
+Rust version **1.92 or later** is required to build certain Python dependencies (such as cryptography) that contain Rust components.
+Homebrew will install the latest stable version of Rust.
+````
+
 Make sure `xcode-select -p` still returns `/Applications/Xcode.app/Contents/Developer`. If that is not the case, run `sudo xcode-select -s /Applications/Xcode.app`
 
 (install_qt)=

docs/build_system/config_windows.md

@@ -74,10 +74,11 @@ All other dependencies are shared across variations.
 4. [Install CMake](install_cmake)
 5. [Install Qt](install_windows_qt)
 6. [Install Strawberry Perl](install_strawberry_perl)
-7. [Install MSYS2](install_msys2)
+7. [Install Rust](install_rust)
+8. [Install MSYS2](install_msys2)
     1. [Install required MSYS2 pacman packages (from an MSYS2-MinGW64 shell)](install_msys2_packages)
-8. [Setup environment variables](setup_env)
-9. [Build Open RV](build_windows_openrv)
+9. [Setup environment variables](setup_env)
+10. [Build Open RV](build_windows_openrv)
 
 
 ````{warning}
@@ -373,8 +374,33 @@ Take note of the installation path for Strawberry Perl, as it will be required i
 The default path is `C:\Strawberry`.
 ````
 
+(install_rust)=
+## 7. Install Rust
+
+````{warning}
+Rust version **1.92 or later** is required to build certain Python dependencies (such as cryptography).
+````
+
+Install Rust using rustup-init:
+
+1. Download rustup-init from [https://win.rustup.rs/x86_64](https://win.rustup.rs/x86_64)
+2. Run `rustup-init.exe`
+3. Follow the prompts to complete the installation (the default options are recommended)
+4. After installation, close and reopen your terminal to apply the PATH changes
+
+Verify that Rust is installed correctly and check the version:
+
+```bash
+rustc --version
+cargo --version
+```
+
+````{note}
+The Rust installation will be located at `%USERPROFILE%\.cargo\bin` (typically `C:\Users\<username>\.cargo\bin`). Do not forget to add the location to the PATH environment variable in [Setup environment variables](#setup_env).
+````
+
 (install_msys2)=
-## 7. Install MSYS2
+## 8. Install MSYS2
 
 ````{warning}
 
@@ -389,7 +415,7 @@ Download and install the latest [MSYS2](https://www.msys2.org/). Open RV is **NO
 MSYS2 is only used for convenience as it comes with a package manager with utility packages required for the Open RV build such as cmake, git, flex, bison, nasm, unzip, zip, etc.
 
 (install_msys2_packages)=
-### 7.1 Install required MSYS2 pacman packages
+### 8.1 Install required MSYS2 pacman packages
 
 ````{note}
 The MSYS2 MingGW64 (mingw64.exe) terminal MUST be used.\
@@ -432,7 +458,7 @@ While installing the MSYS packages, review the list for any missing package. Som
 Note: To confirm which version/location of any tool used inside the MSYS shell, `where` can be used e.g. `where python`. If there's more than one path return, the top one will be used.
 
 (setup_env)=
-### 8. Setup environment variables
+### 9. Setup environment variables
 
 ````{note}
 This is the step where the path of Strawberry Perl, Python, CMake and Qt will be needed.
@@ -447,28 +473,37 @@ These modifications will be added to the `.bash_profile` file located in the Use
 #### PATH environment variable
 
 ````{note}
-Update the CMake, Strawberry Perl and Python location to reflect your installation path, using **forward slashes (/)** for a Unix-style path 
+Update the CMake, Strawberry Perl, Python, and Rust locations to reflect your installation paths, using **forward slashes (/)** for a Unix-style path 
 to prevent issues later on.
 (e.g., C:\Python310 becomes /c/Python310).
+
+**For Rust:** Replace `<username>` with your actual Windows username. You can find your username by running `echo %USERNAME%` in a Windows command prompt.
+The Rust installation is located at `%USERPROFILE%\.cargo\bin` (typically `C:\Users\<username>\.cargo\bin`), 
+which becomes `/c/Users/<username>/.cargo/bin` in MSYS2 format.
 ````
 
 The following paths **must** be added to the PATH environment variable within MSYS2:
 - CMake binary directory
 - Python binary directory
+- Rust cargo binary directory
 - MSYS2's `mingw64/bin`
 - Strawberry perl directory
 
 **The order is important**. Do not put Strawberry perl location before MSYS2's `mingw64/bin` directory.
 
 ````{tabs}
 ```{code-tab} bash VFX-CY2024
-echo 'export PATH="/c/Program Files/CMake/bin:/c/Python311:/c/msys64/mingw64/bin:$PATH:/c/Strawberry/perl/bin"' >> ~/.bash_profile
+echo 'export PATH="/c/Program Files/CMake/bin:/c/Python311:/c/Users/<username>/.cargo/bin:/c/msys64/mingw64/bin:$PATH:/c/Strawberry/perl/bin"' >> ~/.bash_profile
 ```
 ```{code-tab} bash VFX-CY2023
-echo 'export PATH="/c/Program Files/CMake/bin:/c/Python310:/c/msys64/mingw64/bin:$PATH:/c/Strawberry/perl/bin"' >> ~/.bash_profile
+echo 'export PATH="/c/Program Files/CMake/bin:/c/Python310:/c/Users/<username>/.cargo/bin:/c/msys64/mingw64/bin:$PATH:/c/Strawberry/perl/bin"' >> ~/.bash_profile
 ```
 ````
 
+````{warning}
+Remember to replace `<username>` with your actual Windows username in the commands above.
+````
+
 #### ACLOCAL_PATH
 
 ```shell
@@ -501,7 +536,7 @@ source ~/.bash_profile
 ```
 
 (build_windows_openrv)=
-## 9. Build Open RV
+## 10. Build Open RV
 
 Once the platform-specific installation process is complete, building Open RV follows the same process for all platforms. Please refer to the [Common Build Instructions](config_common_build.md) for the complete build process.
 

src/build/requirements.txt.in

@@ -1,10 +1,11 @@
 # This file contains all the packages that will be packaged with RV. Please add the license next to the package.
 # NOTE: This is a CMake template file. The actual requirements.txt is generated during CMake configuration.
 #       To update OpenTimelineIO version, edit _opentimelineio_version in cmake/dependencies/python3.cmake
+# NOTE: Using --no-binary :all: in pip install ensures all packages are built from source against our custom Python.
 
 pip                     # License: MIT License (MIT)
 setuptools              # License: MIT License
-@_opentimelineio_pip_url@ # License: Other/Proprietary License (Modified Apache 2.0 License)
+opentimelineio==@_opentimelineio_version@ # License: Other/Proprietary License (Modified Apache 2.0 License)
 PyOpenGL                # License: BSD License (BSD)
 
 # MacOS only - PyOpenGL_accelerate is built from source in python3.cmake as a workaround until the fix