Merge branch '3.0-dev' into 3.0

Author: jslobodzian

.CodeQL.yml

@@ -0,0 +1,29 @@
+path_classifiers:
+  library:
+    # Treat source files for all compiled languages in the specs directories
+    # as 3rd party library sources because they are not owned by us.
+    #
+    # Extensions from https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/.
+    - "SPECS*/**/*.c"
+    - "SPECS*/**/*.c++"
+    - "SPECS*/**/*.cc"
+    - "SPECS*/**/*.cpp"
+    - "SPECS*/**/*.cs"
+    - "SPECS*/**/*.cshtml"
+    - "SPECS*/**/*.csproj"
+    - "SPECS*/**/*.cts"
+    - "SPECS*/**/*.cxx"
+    - "SPECS*/**/*.go"
+    - "SPECS*/**/*.h"
+    - "SPECS*/**/*.h++"
+    - "SPECS*/**/*.hh"
+    - "SPECS*/**/*.hpp"
+    - "SPECS*/**/*.hxx"
+    - "SPECS*/**/*.java"
+    - "SPECS*/**/*.kt"
+    - "SPECS*/**/*.mts"
+    - "SPECS*/**/*.sln"
+    - "SPECS*/**/*.swift"
+    - "SPECS*/**/*.ts"
+    - "SPECS*/**/*.tsx"
+    - "SPECS*/**/*.xaml"

.github/workflows/check-manifests.yml

@@ -21,7 +21,9 @@ jobs:
     # This PR runner uses an older Ubuntu with rpm version 4.17, which doesn't understand some newer macros like %bcond
     - name: Define missing rpm macros
       run: |
-        [[ -n $(rpm --eval '%bcond test 1') ]] && echo '%bcond() %[ (%{2}) ? "%{expand:%%bcond_without %{1}}" : "%{expand:%%bcond_with %{1}}" ]' > ~/.rpmmacros
+        if [[ -n $(rpm --eval '%bcond test 1') ]]; then
+          echo '%bcond() %[ (%{2}) ? "%{expand:%%bcond_without %{1}}" : "%{expand:%%bcond_with %{1}}" ]' > ~/.rpmmacros
+        fi
 
     - name: Check x86_64 manifests
       run: |

.github/workflows/check-package-cgmanifest.yml

@@ -22,7 +22,9 @@ jobs:
     # This PR runner uses an older Ubuntu with rpm version 4.17, which doesn't understand some newer macros like %bcond
     - name: Define missing rpm macros
       run: |
-        [[ -n $(rpm --eval '%bcond test 1') ]] && echo '%bcond() %[ (%{2}) ? "%{expand:%%bcond_without %{1}}" : "%{expand:%%bcond_with %{1}}" ]' > ~/.rpmmacros
+        if [[ -n $(rpm --eval '%bcond test 1') ]]; then
+          echo '%bcond() %[ (%{2}) ? "%{expand:%%bcond_without %{1}}" : "%{expand:%%bcond_with %{1}}" ]' > ~/.rpmmacros
+        fi
 
     - name: Get base commit for PRs
       if: ${{ github.event_name == 'pull_request' }}

.pipelines/containerSourceData/valkey/Dockerfile-Valkey

@@ -0,0 +1,26 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+ARG BASE_IMAGE
+
+FROM $BASE_IMAGE
+
+@INCLUDE_MAIN_RUN_INSTRUCTION@
+
+RUN set -eux && \
+    valkey-cli --version && \
+    valkey-server --version && \
+    mkdir /data && \
+    chown valkey:valkey /data
+
+VOLUME /data
+WORKDIR /data
+
+COPY valkey-docker-entrypoint.sh /usr/local/bin/
+
+RUN chmod +x /usr/local/bin/valkey-docker-entrypoint.sh
+
+ENTRYPOINT ["/usr/local/bin/valkey-docker-entrypoint.sh"]
+
+EXPOSE 6379
+CMD ["valkey-server"]

.pipelines/containerSourceData/valkey/configuration-files/valkey-docker-entrypoint.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+set -e
+
+# first arg is `-f` or `--some-option`
+# or first arg is `something.conf`
+if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then
+    set -- valkey-server "$@"
+fi
+
+# allow the container to be started with `--user`
+if [ "$1" = 'valkey-server' -a "$(id -u)" = '0' ]; then
+    find . \! -user valkey -exec chown valkey '{}' +
+    exec setpriv --reuid=valkey --regid=valkey --init-groups --inh-caps=-all "$BASH_SOURCE" "$@"
+fi
+
+# set an appropriate umask (if one isn't set already)
+# - https://github.com/docker-library/redis/issues/305
+# - https://github.com/redis/redis/blob/bb875603fb7ff3f9d19aad906bd45d7db98d9a39/utils/systemd-redis_server.service#L37
+um="$(umask)"
+if [ "$um" = '0022' ]; then
+	umask 0077
+fi
+
+exec "$@" $VALKEY_EXTRA_FLAGS

.pipelines/containerSourceData/valkey/valkey.name

@@ -0,0 +1 @@
+valkey

.pipelines/containerSourceData/valkey/valkey.pkg

@@ -0,0 +1,3 @@
+valkey
+cronie
+util-linux

.pipelines/prchecks/PackageBuildPRCheck.yml

@@ -80,9 +80,11 @@ extends:
                       # GCC fails to build as a regular package.
                       ignoredSpecs: ["gcc"]
 
-                  - script: echo "##vso[task.setvariable variable=toolchainArtifactName;isOutput=true]$(ob_artifactBaseName)"
+                  - script: |
+                      echo "##vso[task.setvariable variable=toolchainArtifactName;isOutput=true]$(ob_artifactBaseName)"
+                      echo "##vso[task.setvariable variable=toolchainTarballName;isOutput=true]toolchain_built_rpms_all.tar.gz"
                     name: "ToolchainArtifactName"
-                    displayName: "Set variable for published artifact name"
+                    displayName: "Set variables for published toolchain tarball"
 
                   # 1. Automatic publishing won't work if 'isCustom: true' is set on the pool. We cannot do 'isCustom: false' because
                   #    then OneBranch attempts to perform additional actions (adding build tags for instance), which require additional permissions
@@ -104,24 +106,38 @@ extends:
                   isCustom: true
                   name: ${{ configuration.agentPool }}
                 variables:
+                  inputArtifactsLocation: $(Agent.TempDirectory)
                   ob_artifactBaseName: $(rpmsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
                   ob_outputDirectory: $(Build.ArtifactStagingDirectory)
+                  outputRPMsTarballName: "rpms.tar.gz"
                   toolchainArtifactName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainArtifactName'] ]
+                  toolchainTarballName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainTarballName'] ]
                 steps:
+                  - task: DownloadPipelineArtifact@2
+                    displayName: "Download toolchain"
+                    inputs:
+                      artifact: $(toolchainArtifactName)
+                      patterns: "**/$(toolchainTarballName)"
+                      targetPath: $(inputArtifactsLocation)
+
                   - template: .pipelines/templates/PackageBuild.yml@self
                     parameters:
                       checkBuildRetries: "1"
-                      customToolchainArtifactName: $(toolchainArtifactName)
+                      customToolchainTarballName: $(toolchainTarballName)
+                      inputArtifactsFolder: $(inputArtifactsLocation)
                       isCheckBuild: true
                       isQuickRebuildPackages: true
                       isUseCCache: true
                       maxCPU: "${{ configuration.maxCPUs }}"
                       outputArtifactsFolder: $(ob_outputDirectory)
+                      outputRPMsTarballName: $(outputRPMsTarballName)
                       pipArtifactFeeds: "mariner/Mariner-Pypi-Feed"
                       selfRepoName: self
                       testSuiteName: "[${{ configuration.name }}] Package test"
 
-                  - script: echo "##vso[task.setvariable variable=rpmsArtifactName;isOutput=true]$(ob_artifactBaseName)"
+                  - script: |
+                      echo "##vso[task.setvariable variable=rpmsArtifactName;isOutput=true]$(ob_artifactBaseName)"
+                      echo "##vso[task.setvariable variable=rpmsTarballName;isOutput=true]$(outputRPMsTarballName)"
                     name: "RPMsArtifactName"
                     displayName: "Set variable for published artifact name"
 
@@ -142,15 +158,25 @@ extends:
                   isCustom: true
                   name: ${{ configuration.agentPool }}
                 variables:
+                  inputArtifactsLocation: $(Agent.TempDirectory)
                   ob_artifactBaseName: $(toolchainTestsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
                   ob_outputDirectory: $(Build.ArtifactStagingDirectory)
                   testListFromToolchain: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['CalculateToolchainPackageRetestList.toolchainPackageRetestList'] ]
                   toolchainArtifactName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainArtifactName'] ]
+                  toolchainTarballName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainTarballName'] ]
                 steps:
+                  - task: DownloadPipelineArtifact@2
+                    displayName: "Download toolchain"
+                    inputs:
+                      artifact: $(toolchainArtifactName)
+                      patterns: "**/$(toolchainTarballName)"
+                      targetPath: $(inputArtifactsLocation)
+
                   - template: .pipelines/templates/PackageBuild.yml@self
                     parameters:
                       checkBuildRetries: "1"
-                      customToolchainArtifactName: $(toolchainArtifactName)
+                      customToolchainTarballName: $(toolchainTarballName)
+                      inputArtifactsFolder: $(inputArtifactsLocation)
                       isAllowToolchainRebuilds: true
                       isCheckBuild: true
                       isQuickRebuildPackages: true
@@ -179,8 +205,18 @@ extends:
                   isCustom: true
                   name: ${{ configuration.agentPool }}
                 variables:
+                  inputArtifactsLocation: $(Agent.TempDirectory)
                   rpmsArtifactName: $[ stageDependencies.RPMs_${{ configuration.name }}.BuildAndTest.outputs['RPMsArtifactName.rpmsArtifactName'] ]
+                  rpmsTarballName: $[ stageDependencies.RPMs_${{ configuration.name }}.BuildAndTest.outputs['RPMsArtifactName.rpmsTarballName'] ]
                 steps:
+                  - task: DownloadPipelineArtifact@2
+                    displayName: "Download RPMs tarball"
+                    inputs:
+                      artifact: $(rpmsArtifactName)
+                      patterns: "**/$(rpmsTarballName)"
+                      targetPath: $(inputArtifactsLocation)
+
                   - template: .pipelines/templatesWithCheckout/SodiffCheck.yml@self
                     parameters:
-                      inputArtifactName: $(rpmsArtifactName)
+                      inputArtifactsFolder: $(inputArtifactsLocation)
+                      inputRPMsTarballName: $(rpmsTarballName)

.pipelines/templates/PackageBuild.yml

@@ -14,13 +14,9 @@ parameters:
     type: number
     default: 12
 
-  - name: customToolchainArtifactName
-    type: string
-    default: ""
-
   - name: customToolchainTarballName
     type: string
-    default: "toolchain_built_rpms_all.tar.gz"
+    default: ""
 
   - name: extraPackageRepos
     type: string
@@ -30,12 +26,16 @@ parameters:
     type: boolean
     default: true
 
-  - name: inputCacheArtifacts
+  - name: inputArtifactsFolder
+    type: string
+    default: "$(Agent.TempDirectory)"
+
+  - name: inputCacheRPMsTarballs
     type: object
     default: []
     # Sample:
-    # - name: build-artifacts
-    #   rpmsTarball: cache.tar.gz
+    #   - cache.tar.gz
+    #   - cache2.tar.gz
 
   - name: isAllowToolchainRebuilds
     type: string
@@ -160,15 +160,9 @@ steps:
           artifactFeeds: "${{ parameters.pipArtifactFeeds }}"
         displayName: "Authenticate to custom pip artifact feeds"
 
-  - ${{ if parameters.customToolchainArtifactName }}:
-      - task: DownloadPipelineArtifact@2
-        displayName: "Download toolchain"
-        inputs:
-          artifact: "${{ parameters.customToolchainArtifactName }}"
-          patterns: "**/${{ parameters.customToolchainTarballName }}"
-
+  - ${{ if parameters.customToolchainTarballName }}:
       - script: |
-          toolchain_archive="$(find "$(Pipeline.Workspace)" -name "${{ parameters.customToolchainTarballName }}" -print -quit)"
+          toolchain_archive="$(find "${{ parameters.inputArtifactsFolder }}" -name "${{ parameters.customToolchainTarballName }}" -print -quit)"
           if [[ ! -f "$toolchain_archive" ]]; then
             echo "ERROR: toolchain archive not found!" >&2
             exit 1
@@ -178,17 +172,11 @@ steps:
           sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" toolchain TOOLCHAIN_ARCHIVE="$toolchain_archive"
         displayName: "Populate toolchain"
 
-  - ${{ each inputCacheArtifact in parameters.inputCacheArtifacts }}:
-      - task: DownloadPipelineArtifact@2
-        displayName: "Download input cache RPM from ${{ inputCacheArtifact.name }}"
-        inputs:
-          artifact: "${{ inputCacheArtifact.name }}"
-          patterns: "**/${{ inputCacheArtifact.rpmsTarball }}"
-
+  - ${{ each inputCacheRPMsTarball in parameters.inputCacheRPMsTarballs }}:
       - script: |
-          rpms_archive="$(find "$(Pipeline.Workspace)" -name "${{ inputCacheArtifact.rpmsTarball }}" -print -quit)"
+          rpms_archive="$(find "${{ parameters.inputArtifactsFolder }}" -name "${{ inputCacheRPMsTarball }}" -print -quit)"
           if [[ ! -f "$rpms_archive" ]]; then
-            echo "ERROR: cache RPMs archive '${{ inputCacheArtifact.rpmsTarball }}' not found!" >&2
+            echo "ERROR: cache RPMs archive '${{ inputCacheRPMsTarball }}' not found!" >&2
             exit 1
           fi
 
@@ -200,7 +188,7 @@ steps:
         check_build_retries_arg="CHECK_BUILD_RETRIES=${{ parameters.checkBuildRetries }}"
       fi
 
-      if [[ -n "${{ parameters.customToolchainArtifactName }}" ]]; then
+      if [[ -n "${{ parameters.customToolchainTarballName }}" ]]; then
         toolchain_archive_arg="TOOLCHAIN_ARCHIVE=$(toolchainArchive)"
       fi
 

.pipelines/templatesWithCheckout/SodiffCheck.yml

@@ -6,8 +6,9 @@ parameters:
     type: string
     default: "$(Build.SourcesDirectory)"
 
-  - name: inputArtifactName
+  - name: inputArtifactsFolder
     type: string
+    default: "$(Agent.TempDirectory)"
 
   - name: inputRPMsTarballName
     type: string
@@ -26,19 +27,11 @@ parameters:
     default: "$(Agent.TempDirectory)/SourcesWorkspace"
 
 steps:
-  - task: DownloadPipelineArtifact@2
-    displayName: "Download sources for signing"
-    inputs:
-      artifact: ${{ parameters.inputArtifactName }}
-      patterns: |
-        **/${{ parameters.inputRPMsTarballName }}
-      targetPath: "$(Agent.TempDirectory)"
-
   - script: |
       set -e
 
       mkdir -p "${{ parameters.sourcesWorkspace }}"
-      find "$(Agent.TempDirectory)" -name "${{ parameters.inputRPMsTarballName }}" -print0 | xargs -0 -n 1 tar -C "${{ parameters.sourcesWorkspace }}" -xkf
+      find "${{ parameters.inputArtifactsFolder }}" -name "${{ parameters.inputRPMsTarballName }}" -print0 | xargs -0 -n 1 tar -C "${{ parameters.sourcesWorkspace }}" -xkf
     displayName: "Extract sources tarball"
 
   - script: |