Merged PR 10406838: Add OpenSSL implementation for XtsAes test

## Description:

We add OpenSSL as submodule to 3rdparty and link symcryptunittest to it so we can compare the SymCrypt implementation. We add perf and functional test for XtsAes as well.

## Admin Checklist:
- [ ] You have updated documentation in symcrypt.h to reflect any changes in behavior
- [ ] You have updated CHANGELOG.md to reflect any changes in behavior
- [ ] You have updated symcryptunittest to exercise any new functionality
- [ ] If you have introduced any symbols in symcrypt.h you have updated production and test dynamic export symbols (exports.ver / exports.def / symcrypt.src) and tested the updated dynamic modules with symcryptunittest
- [ ] If you have introduced functionality that varies based on CPU features, you have manually tested with and without relevant features
- [ ] If you have made significant changes to a particular algorithm, you have checked that performance numbers reported by symcryptunittest are in line with expectations
- [ ] If you have added new algorithms/modes, you have updated the status indicator text for the associated modules if necessary

Related work items: #49347468

Author: yourmsftacct

.pipelines/OneBranch.PullRequest.yml

@@ -16,7 +16,7 @@ variables:
   WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2019/vse2022@sha256:a3083215a4675bad774ec88005dcf57436b3423584d0db3e82b6de3f780213ba'
 
 resources:
-  repositories: 
+  repositories:
     - repository: templates
       type: git
       name: OneBranch.Pipelines/GovernedTemplates
@@ -55,7 +55,7 @@ extends:
           inputs:
             system: 'Custom'
             customVersion: '$(verStep.VER_MAJOR).$(verStep.VER_MINOR).$(verStep.VER_PATCH)-$(Build.BuildId)'
-          
+
     - stage: Build_Windows
       displayName: Build Windows
       jobs:
@@ -103,44 +103,50 @@ extends:
           config: 'Debug'
           cc: 'gcc'
           cxx: 'g++'
+          openssl: true
       - template: .pipelines/templates/build-linux.yml@self
         parameters:
           arch: 'AMD64'
           config: 'Sanitize'
           cc: 'gcc'
           cxx: 'g++'
+          openssl: true
       - template: .pipelines/templates/build-linux.yml@self
         parameters:
           arch: 'AMD64'
           config: 'Release'
           cc: 'gcc'
-          cxx: 'g++'          
+          cxx: 'g++'
+          openssl: true
       - template: .pipelines/templates/build-linux.yml@self
         parameters:
           arch: 'AMD64'
           config: 'Debug'
           cc: 'clang'
           cxx: 'clang++'
+          openssl: true
       - template: .pipelines/templates/build-linux.yml@self
         parameters:
           arch: 'AMD64'
           config: 'Sanitize'
           cc: 'clang'
           cxx: 'clang++'
+          openssl: true
       - template: .pipelines/templates/build-linux.yml@self
         parameters:
           arch: 'AMD64'
           config: 'Release'
           cc: 'clang'
           cxx: 'clang++'
+          openssl: true
       - template: .pipelines/templates/build-linux.yml@self
         parameters:
           arch: 'AMD64'
           config: 'Release'
           cc: 'gcc'
           cxx: 'g++'
           skipTests: true
-          additionalArgs: '--no-asm'
+          additionalArgs: '--no-asm --openssl'
           identifier: 'NoAsm'
       - template: .pipelines/templates/build-linux.yml@self
         parameters:
@@ -149,7 +155,7 @@ extends:
           cc: 'clang'
           cxx: 'clang++'
           skipTests: true
-          additionalArgs: '--no-asm'
+          additionalArgs: '--no-asm --openssl'
           identifier: 'NoAsm'
       - template: .pipelines/templates/build-linux.yml@self
         parameters:

.pipelines/templates/build-linux.yml

@@ -31,6 +31,9 @@ parameters:
 - name: identifier # Additional identifier for job name
   type: string
   default: ''
+- name: openssl # Build with OpenSSL
+  type: boolean
+  default: false
 
 jobs:
 - job: Linux_${{ parameters.arch}}_${{ parameters.cc }}_${{parameters.config}}_${{parameters.identifier}}
@@ -46,39 +49,47 @@ jobs:
       verbose_build_flag: '--verbose'
     ${{ else }}:
       verbose_build_flag: ''
+    ${{ if eq(parameters.openssl, true) }}:
+      openssl_build_flag: '--openssl'
+    ${{ else }}:
+      openssl_build_flag: ''
 
   steps:
+    - script: |
+        apt-get update -y
+      displayName: 'Update package manager'
 
     - ${{ if eq(parameters.cc, 'clang') }}:
       # Install clang-11 and manually add symlinks so that it can be run via just "clang" rather
       # than clang-11. This is required because we can't explicitly specify clang-11 for
       # parameters.cc (it breaks the job name, because job names aren't allowed to have '-' in them)
       - script: |
-          apt-get update
           apt-get install -y clang-11
           ln -s /usr/bin/clang-11 /usr/bin/clang
           ln -s /usr/bin/clang++-11 /usr/bin/clang++
         displayName: 'Install clang'
 
     - ${{ if eq(parameters.arch, 'X86') }}:
       - script: |
-          apt-get update
           apt-get install -y gcc-multilib g++-multilib
         displayName: 'Install x86 headers and libraries'
 
     # Note: this assumes that the pipeline is always running on an AMD64 machine. When we have
     # native ARM64 pipelines, we'll need to change this.
     - ${{ if eq(parameters.arch, 'ARM64') }}:
       - script: |
-          apt-get update
           apt-get install -y binutils-aarch64-linux-gnu gcc-aarch64-linux-gnu g++-aarch64-linux-gnu qemu-user
         displayName: 'Install arm64 cross-compilation tools'
 
     - ${{ if eq(parameters.arch, 'ARM') }}:
       - script: |
-          apt-get update
           apt-get install -y gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf qemu-user
-        displayName: 'Install arm64 cross-compilation tools'
+        displayName: 'Install arm cross-compilation tools'
+
+    - ${{ if eq(parameters.openssl, true) }}:
+      - script: |
+          apt-get install -y libssl-dev
+        displayName: 'Install OpenSSL'
 
     - task: PipAuthenticate@1
       inputs:
@@ -93,14 +104,14 @@ jobs:
       inputs:
         scriptSource: 'filePath'
         scriptPath: scripts/build.py
-        arguments: 'cmake bin --arch ${{ parameters.arch }} --config ${{ parameters.config }} --cc ${{ parameters.cc }} --cxx ${{ parameters.cxx }} ${{ parameters.additionalArgs }} $(verbose_build_flag)'
+        arguments: 'cmake bin --arch ${{ parameters.arch }} --config ${{ parameters.config }} --cc ${{ parameters.cc }} --cxx ${{ parameters.cxx }} ${{ parameters.additionalArgs }} $(verbose_build_flag) $(openssl_build_flag)'
         workingDirectory: $(Build.SourcesDirectory)
-  
+
     # Overwrite default artifact publishing with our copy (enables publishing binaries for failed runs)
     - script: |
           cp .artifactignore "$(ob_outputDirectory)/.artifactignore"
       displayName: 'Overwrite .artifactignore'
-    
+
     - ${{ if ne(parameters.skipTests, true) }}:
       - ${{ if ne(parameters.arch, 'ARM64') }}:
         - ${{ if ne(parameters.arch, 'ARM') }}:
@@ -146,7 +157,7 @@ jobs:
             scriptPath: scripts/test.py
             arguments: '--emulator qemu-aarch64 --emulator-lib-dir /usr/aarch64-linux-gnu/ bin dynamic:bin/module/generic/libsymcrypt.so noperftests +symcrypt -dh -dsa -rsa'
             workingDirectory: $(Build.SourcesDirectory)
-      
+
       - ${{ if eq(parameters.arch, 'ARM') }}:
         - task: PythonScript@0
           displayName: 'Run unit tests'
@@ -170,4 +181,4 @@ jobs:
         scriptSource: 'filePath'
         scriptPath: scripts/package.py
         arguments: 'bin ${{ parameters.arch }} ${{ parameters.config }} generic bin'
-        workingDirectory: $(Build.SourcesDirectory) 
+        workingDirectory: $(Build.SourcesDirectory)

BUILD.md

@@ -60,7 +60,7 @@ Linux modules with FIPS integrity checks.
     * `-DSYMCRYPT_TARGET_ARCH=<AMD64|X86|ARM64>` to choose a target architecture. If not specified, it will default to the host system architecture.
       * To cross-compile for Windows X86 from Windows AMD64, you must also use `-A Win32`
       * To cross-compile for Linux ARM64, you must also use `--toolchain=cmake-configs/Toolchain-Clang-ARM64.cmake`
-    * `-DSYMCRYPT_USE_ASM=<ON|OFF>` to choose whether to use assembly optimizations. Defaults to `ON`. 
+    * `-DSYMCRYPT_USE_ASM=<ON|OFF>` to choose whether to use assembly optimizations. Defaults to `ON`.
     * `-DSYMCRYPT_FIPS_BUILD=<ON|OFF>` to choose whether to enable FIPS self-tests in the SymCrypt shared object module. Defaults to `ON`. Currently only affects Linux builds.
     * For a release build, specify `-DCMAKE_BUILD_TYPE=RelWithDebInfo`
 1. `cmake --build bin`
@@ -85,19 +85,39 @@ outputs are placed in this directory.
 
 Requires the following packages on debian-based systems to build:
 ```
-apt-get -y install --no-install-recommends \
+apt update
+apt -y install --no-install-recommends \
     cmake \
-    python3-pyelftools \                             # integrity
-    gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf  # for arm
+    python3-pyelftools \
+    gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf
 ```
-
+`python3-pyelftools` is for integrity verification and `gcc-arm-linux-gnueabihf` `g++-arm-linux-gnueabihf` are for ARM cross compile
 And for running the test:
 ```
-apt-get -y install --no-install-recommends qemu-user
+apt -y install --no-install-recommends qemu-user
 ```
 
 To build and test for example for arm:
 ```
 python3 scripts/build.py cmake --arch arm --toolchain cmake-configs/Toolchain-GCC-ARM.cmake bin_arm
 qemu-arm -L /usr/arm-linux-gnueabihf/ ./bin_arm/exe/symcryptunittest -rsa -dsa -dh -ec -int -mod dynamic:bin_arm/module/generic/libsymcrypt.so
-```
+```
+
+## Performance comparison with OpenSSL
+`symcryptunittest.exe` can be used to compare and measure performance of algorithms provided by SymCrypt and OpenSSL. On Windows `symcryptunittest.exe` would have to be compiled with OpenSSL. `nasm` and  `strawberryperl` are prerequisites to building OpenSSL.
+```
+winget install nasm strawberryperl
+.\scripts\build.py cmake bin --config Release --openssl-build-from-source
+```
+
+And on Linux we can use OpenSSL installed by system's package manager.
+```
+sudo apt install -y libssl-dev
+./scripts/build.py cmake bin --config Release --openssl
+```
+
+To build OpenSSL on Linux we need to install following prerequisites.
+```
+sudo apt install -y nasm perl
+.\scripts\build.py cmake bin --config Release --openssl-build-from-source
+```

CHANGELOG.md

@@ -4,6 +4,8 @@ New changes will be listed here as they are developed. The version number is det
 prior to the creation of a new release, based on the changes contained in that release.
 
 - Add SymCryptEntropyAccumulator to Windows kernel module
+- Fix tweak lower 64 bit overflow calculation in SYMCRYPT_XtsAesXxx
+- Add OpenSSL implementation for XtsAes to symcryptunittest
 
 # Version 103.4.1
 - Add retpoline guard flags for undocked Windows build

CMakeLists.txt

@@ -58,7 +58,7 @@ option(
     SYMCRYPT_TEST_LEGACY_IMPL
     "When enabled, the SymCrypt unit tests will be linked against and configured to run compatibility and performance tests on the legacy
     RSA32 and msbignum cryptographic implementations. This requires access to private static libraries which are not licensed
-    for use outside of Windows, and thus can only be used by Microsoft employees building internally. It only affects the unit tests, 
+    for use outside of Windows, and thus can only be used by Microsoft employees building internally. It only affects the unit tests,
     and does not change the functionality of the SymCrypt library itself."
     OFF)
 
@@ -74,10 +74,20 @@ option(
     ON, but may need to be disabled for specialized targets such as embedded systems."
     ON)
 
+option(
+    SYMCRYPT_TEST_WITH_OPENSSL
+    "When enabled, the SymCrypt unit tests will be linked against and configured to run performance comparisons against OpenSSL
+    implementations of certain algorithms."
+    OFF)
+
+
+if (SYMCRYPT_TEST_WITH_OPENSSL)
+    include(${CMAKE_SOURCE_DIR}/cmake-configs/OpenSSL.cmake)
+endif()
 include(${CMAKE_SOURCE_DIR}/cmake-configs/SymCrypt-Platforms.cmake)
 
 if(NOT DEFINED CMAKE_BUILD_TYPE)
-  set(CMAKE_BUILD_TYPE Debug)
+    set(CMAKE_BUILD_TYPE Debug)
 endif()
 
 set(CMAKE_C_STANDARD 11)

cmake-configs/OpenSSL.cmake

@@ -0,0 +1,71 @@
+if(OPENSSL_BUILD_FROM_SOURCE)
+    if(CMAKE_BUILD_TYPE MATCHES "Release|RelWithDebInfo")
+        set(OPENSSL_BUILD_TYPE release)
+    else()
+        set(OPENSSL_BUILD_TYPE debug)
+    endif()
+    if(NOT DEFINED OPENSSL_BUILD_BRANCH)
+        set(OPENSSL_BUILD_BRANCH_SUFFIX "")
+    else()
+        set(OPENSSL_BUILD_BRANCH_SUFFIX -${OPENSSL_BUILD_BRANCH})
+    endif()
+
+    set(OPENSSL_BUILD_ROOT ${CMAKE_SOURCE_DIR}/3rdparty/openssl-${OPENSSL_BUILD_TYPE}${OPENSSL_BUILD_BRANCH_SUFFIX})
+
+    if(NOT IS_DIRECTORY "${OPENSSL_BUILD_ROOT}" OR NOT EXISTS "${OPENSSL_BUILD_ROOT}/Configure")
+        execute_process(
+            COMMAND git clone https://github.com/openssl/openssl.git ${OPENSSL_BUILD_ROOT}
+            WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+        if(NOT OPENSSL_BUILD_BRANCH STREQUAL "")
+            execute_process(
+                COMMAND git checkout ${OPENSSL_BUILD_BRANCH}
+                WORKING_DIRECTORY ${OPENSSL_BUILD_ROOT})
+        endif()
+    endif()
+
+    if(NOT EXISTS "${OPENSSL_BUILD_ROOT}/OpenSSLConfig.cmake")
+        set(ENV{LANG} C)
+        set(ENV{LC_ALL} C)
+        set(ENV{CL} /MP)
+        if(OPENSSL_BUILD_TYPE STREQUAL "release")
+            execute_process(
+                COMMAND perl Configure no-ssl no-tls1 no-tls1_1 --release
+                WORKING_DIRECTORY ${OPENSSL_BUILD_ROOT}
+                RESULT_VARIABLE result)
+        else()
+            execute_process(
+                COMMAND perl Configure no-ssl no-tls1 no-tls1_1 --debug
+                WORKING_DIRECTORY ${OPENSSL_BUILD_ROOT}
+                RESULT_VARIABLE result)
+        endif()
+        if(CMAKE_SYSTEM_NAME MATCHES "Linux")
+            cmake_host_system_information(RESULT J
+                QUERY NUMBER_OF_LOGICAL_CORES)
+
+            execute_process(
+                COMMAND make -j ${J}
+                WORKING_DIRECTORY ${OPENSSL_BUILD_ROOT}
+                RESULT_VARIABLE result)
+        elseif(CMAKE_SYSTEM_NAME MATCHES "Windows")
+            execute_process(
+                COMMAND nmake
+                WORKING_DIRECTORY ${OPENSSL_BUILD_ROOT}
+                RESULT_VARIABLE result)
+        else()
+            message(FATAL_ERROR "Unsupported platform")
+        endif()
+    endif()
+
+    # If you don't static link, symcryptunittest.exe may use C:\WINDOWS\SYSTEM32\libcrypto-3-x64.dll
+    set(OPENSSL_USE_STATIC_LIBS True)
+
+    include(${OPENSSL_BUILD_ROOT}/OpenSSLConfig.cmake)
+else()
+    find_package(OpenSSL REQUIRED)
+endif()
+
+message("Found OpenSSL include directory ${OPENSSL_INCLUDE_DIR}")
+include_directories(${OPENSSL_INCLUDE_DIR})
+link_directories(${OPENSSL_LIBRARY_DIR})
+link_libraries(${OPENSSL_CRYPTO_LIBRARIES})
+add_compile_options(-DINCLUDE_IMPL_OPENSSL=1)

lib/CMakeLists.txt

@@ -163,7 +163,7 @@ function(process_cppasm filepath outformat archdefine)
         # assume masm => MSVC C compiler
         add_custom_command(
             OUTPUT ${output_asm}
-            COMMAND "${CMAKE_C_COMPILER}" /EP /P /Fi${output_asm} ${filepath}
+            COMMAND "${CMAKE_C_COMPILER}" /nologo /EP /P /Fi${output_asm} ${filepath}
                 -I${CMAKE_CURRENT_SOURCE_DIR} -I${CMAKE_CURRENT_SOURCE_DIR}/${rootpath} -I${CMAKE_SOURCE_DIR}/inc
                 -DSYMCRYPT_${outformatupper} -DSYMCRYPT_CPU_${archdefineupper} ${dbg_definition}
             MAIN_DEPENDENCY ${filepath}

lib/xtsaes_pattern.c

@@ -57,9 +57,9 @@ SYMCRYPT_XtsAesXxx(
             // bOverflow=FALSE allows backwards compatibility with old API which wrapped around at 64-bits
             SYMCRYPT_ASSERT( tweakLow64 < N_PARALLEL_TWEAKS );
 
-            // Increment tweakHigh64 and store new value in high half of the previous tweakLow64+1 tweaks
+            // Increment tweakHigh64 and store new value in high half of the previous tweakLow64 tweaks
             tweakHigh64++;
-            for( i=0; i<=tweakLow64; i++)
+            for( i=0; i<tweakLow64; i++)
             {
                 SYMCRYPT_STORE_LSBFIRST64(&tweakBuf[tweakBytes - (16*i) - 8], tweakHigh64);
             }

modules/linux/common/CMakeLists.txt

@@ -19,6 +19,12 @@ if (CMAKE_C_COMPILER_ID MATCHES "Clang" AND DEFINED CMAKE_C_COMPILER_TARGET)
   set(jitter_ldflags "--target=${CMAKE_C_COMPILER_TARGET}")
 endif()
 
+if(NOT IS_DIRECTORY "${CMAKE_SOURCE_DIR}/3rdparty/jitterentropy-library" OR NOT EXISTS "${CMAKE_SOURCE_DIR}/3rdparty/jitterentropy-library/Makefile")
+  execute_process(
+    COMMAND git submodule update --init -- 3rdparty/jitterentropy-library
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+endif()
+
 add_custom_target(jitterentropy_lib ALL
   COMMAND make clean
   COMMAND ${CMAKE_COMMAND} -E
@@ -27,4 +33,4 @@ add_custom_target(jitterentropy_lib ALL
     env "LDFLAGS=${jitter_ldflags}"
     make
   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/jitterentropy-library
-)
+)