Merge remote-tracking branch 'upstream/main' into wangamber/transcription

Author: amber-yujueWang

eng/automation/parameters.py

@@ -17,7 +17,7 @@
 
 SDK_ROOT = "../../"  # related to file dir
 AUTOREST_CORE_VERSION = "3.9.7"
-AUTOREST_JAVA = "@autorest/[email protected].60"
+AUTOREST_JAVA = "@autorest/[email protected].62"
 DEFAULT_VERSION = "1.0.0-beta.1"
 GROUP_ID = "com.azure.resourcemanager"
 API_SPECS_FILE = "api-specs.yaml"

eng/common/pipelines/ai-evals-tests.yml

@@ -15,83 +15,34 @@ parameters:
     default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations'
   - name: OpenAIEndPoint
     type: string
-    default: 'https://ai-prmarottai3149546654251245.openai.azure.com/'
+    default: 'https://openai-shared.openai.azure.com/'
   - name: Model
     type: string
     default: 'gpt-5'
-  - name: ToolsRepoName
+  - name: EvalRepoOwner
+    type: string
+    default: 'Azure'
+  - name: EvalRepoName
     type: string
     default: 'azure-sdk-tools'
-  - name: ToolsRepoOwner
+  - name: EvalRepoCommit
     type: string
-    default: 'Azure'
-
-jobs:
-  - job: Run_Eval
-    variables:
-      - template: /eng/pipelines/templates/variables/globals.yml
-      - template: /eng/pipelines/templates/variables/image.yml
-    displayName: 'Run AI Eval'
-    pool:
-      name: $(LINUXPOOL)
-      image: $(LINUXVMIMAGE)
-      os: linux
-
-    steps:
-    - task: UseDotNet@2
-      displayName: "Use .NET SDK 9.0.x"
-      retryCountOnTaskFailure: 3
-      inputs:
-        packageType: sdk
-        version: 9.0.x
-        performMultiLevelLookup: true
-
-    - task: UseDotNet@2
-      displayName: "Use .NET SDK 8.0.x"
-      retryCountOnTaskFailure: 3
-      inputs:
-        packageType: sdk
-        version: 8.0.x
-        performMultiLevelLookup: true
-
-    - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
-      parameters:
-        SkipCheckoutNone: true
-        Repositories:
-          - Name: $(Build.Repository.Name)
-            Commitish: $(Build.SourceVersion)
-            WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)
-          - Name: ${{ parameters.ToolsRepoOwner }}/${{ parameters.ToolsRepoName }}
-            WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.ToolsRepoName }}
-        Paths:
-          - 'tools/**'
-          - eng/common/**
-          - .github/copilot-instructions.md
-    
-    - task: AzureCLI@2
-      displayName: 'Run eval'
-      inputs:
-        azureSubscription: opensource-api-connection
-        scriptType: 'bash'
-        scriptLocation: 'inlineScript'
-        workingDirectory: '${{ parameters.ToolsRepoName }}/${{ parameters.EvalProject }}'
-        inlineScript: |
-          echo "Logged in to Azure"
-          echo "Running eval in project ${{ parameters.ToolsRepoName }}/${{ parameters.EvalProject }}"
-          dotnet test /p:ArtifactsPackagesDir=$(Build.ArtifactStagingDirectory) --logger trx
-      env:
-        DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
-        DOTNET_CLI_TELEMETRY_OPTOUT: 1
-        DOTNET_MULTILEVEL_LOOKUP: 0
-        AZURE_OPENAI_MODEL_DEPLOYMENT_NAME: ${{ parameters.Model }}
-        AZURE_OPENAI_ENDPOINT: ${{ parameters.OpenAIEndPoint }}
-        REPOSITORY_NAME: $(Build.Repository.Name)
-        COPILOT_INSTRUCTIONS_PATH_MCP_EVALS: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/.github/copilot-instructions.md
+    default: 'main'
 
-    - task: PublishTestResults@2
-      condition: succeededOrFailed()
-      inputs:
-        testResultsFiles: '**/*.trx'
-        testRunTitle: $(System.JobDisplayName)
-        testResultsFormat: 'VSTest'
-        mergeTestResults: true
+variables:
+  TargetRepoOwner: ${{ split(variables['Build.Repository.Name'], '/')[0] }}
+  TargetRepoName: ${{ split(variables['Build.Repository.Name'], '/')[1] }}
+  TargetRepoCommit: ${{ variables['Build.SourceVersion'] }}
+ 
+extends:
+  template: /eng/common/pipelines/templates/jobs/ai-eval-job.yml
+  parameters:
+    EvalProject: ${{ parameters.EvalProject }}
+    OpenAIEndPoint: ${{ parameters.OpenAIEndPoint }}
+    Model: ${{ parameters.Model }}
+    EvalRepoOwner: ${{ parameters.EvalRepoOwner }}
+    EvalRepoName: ${{ parameters.EvalRepoName }}
+    EvalRepoCommit: ${{ parameters.EvalRepoCommit }}
+    TargetRepoOwner: $(TargetRepoOwner)
+    TargetRepoName: $(TargetRepoName)
+    TargetRepoCommit: $(TargetRepoCommit)

eng/common/pipelines/templates/jobs/ai-eval-job.yml

@@ -0,0 +1,100 @@
+parameters:
+  - name: EvalProject
+    type: string
+    default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations'
+  - name: OpenAIEndPoint
+    type: string
+    default: 'https://openai-shared.openai.azure.com/'
+  - name: Model
+    type: string
+    default: 'gpt-5'
+# Repository where evals are located
+  - name: EvalRepoOwner
+    type: string
+    default: 'Azure'
+  - name: EvalRepoName
+    type: string
+    default: 'azure-sdk-tools'
+  - name: EvalRepoCommit
+    type: string
+    default: 'main'
+# Target repository for copilot instructions
+  - name: TargetRepoOwner
+    type: string
+  - name: TargetRepoName
+    type: string
+  - name: TargetRepoCommit
+    type: string
+
+jobs:
+  - job: Run_Eval
+    variables:
+      - template: /eng/pipelines/templates/variables/globals.yml
+      - template: /eng/pipelines/templates/variables/image.yml
+    displayName: 'Run AI Eval'
+    pool:
+      name: $(LINUXPOOL)
+      image: $(LINUXVMIMAGE)
+      os: linux
+
+    steps:
+    - checkout: none
+    - task: UseDotNet@2
+      displayName: "Use .NET SDK 9.0.x"
+      retryCountOnTaskFailure: 3
+      inputs:
+        packageType: sdk
+        version: 9.0.x
+        performMultiLevelLookup: true
+
+    - task: UseDotNet@2
+      displayName: "Use .NET SDK 8.0.x"
+      retryCountOnTaskFailure: 3
+      inputs:
+        packageType: sdk
+        version: 8.0.x
+        performMultiLevelLookup: true
+
+    - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+      parameters:
+        SkipCheckoutNone: true
+        Repositories:
+          - Name: ${{ parameters.EvalRepoOwner }}/${{ parameters.EvalRepoName }}
+            Commitish: ${{ parameters.EvalRepoCommit }}
+            WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.EvalRepoName }}
+          - Name: ${{ parameters.TargetRepoOwner }}/${{ parameters.TargetRepoName }}
+            Commitish: ${{ parameters.TargetRepoCommit }}
+            WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.TargetRepoName }}
+        Paths:
+          - 'tools/**'
+          - 'eng/common/**'
+          - '.github/copilot-instructions.md'
+    
+    - task: AzureCLI@2
+      displayName: 'Run eval'
+      condition: and(succeeded(), ne(variables['Skip.Eval'], 'true'))
+      inputs:
+        azureSubscription: opensource-api-connection
+        scriptType: 'bash'
+        scriptLocation: 'inlineScript'
+        workingDirectory: '$(System.DefaultWorkingDirectory)/${{ parameters.EvalRepoName }}/${{ parameters.EvalProject }}'
+        inlineScript: |
+          echo "Logged in to Azure"
+          echo "Running eval"
+          dotnet test --logger trx
+      env:
+        DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+        DOTNET_CLI_TELEMETRY_OPTOUT: 1
+        DOTNET_MULTILEVEL_LOOKUP: 0
+        AZURE_OPENAI_MODEL_DEPLOYMENT_NAME: ${{ parameters.Model }}
+        AZURE_OPENAI_ENDPOINT: ${{ parameters.OpenAIEndPoint }}
+        REPOSITORY_NAME: ${{ parameters.TargetRepoName }}
+        COPILOT_INSTRUCTIONS_PATH_MCP_EVALS: $(System.DefaultWorkingDirectory)/${{ parameters.TargetRepoName }}/.github/copilot-instructions.md
+
+    - task: PublishTestResults@2
+      condition: succeededOrFailed()
+      inputs:
+        testResultsFiles: '**/*.trx'
+        testRunTitle: $(System.JobDisplayName)
+        testResultsFormat: 'VSTest'
+        mergeTestResults: true

eng/common/pipelines/templates/steps/daily-dev-build-variable.yml

@@ -2,6 +2,7 @@
 # is used when this pipeline is going to be generating and publishing daily dev builds.
 parameters:
   ServiceDirectory: ''
+  Artifacts: []
   Condition: succeeded()
 steps:
 - ${{if ne(parameters.ServiceDirectory, '')}}:
@@ -11,6 +12,7 @@ steps:
       arguments: >
         -ServiceDirectory ${{parameters.ServiceDirectory}}
         -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo
+        -artifactList @('${{ replace(convertToJson(parameters.Artifacts), '''', '`''') }}' | ConvertFrom-Json | Select-Object -ExpandProperty name)
       pwsh: true
       workingDirectory: $(Pipeline.Workspace)
     displayName: Dump Package properties

eng/common/scripts/Save-Package-Properties.ps1

@@ -30,7 +30,11 @@ package properties JSON file. If the package properties JSON file already
 exists, read the Version property from the existing package properties JSON file
 and set that as the Version property for the new output. This has the effect of
 "adding" a DevVersion property to the file which could be different from the
-Verison property in that file.
+Version property in that file.
+
+.PARAMETER artifactList
+Optional array of artifact names to filter the package properties. Only packages
+with artifact names matching entries in this list will be processed.
 #>
 
 [CmdletBinding()]
@@ -39,7 +43,8 @@ Param (
   [Parameter(Mandatory = $True)]
   [string] $outDirectory,
   [string] $prDiff,
-  [switch] $addDevVersion
+  [switch] $addDevVersion,
+  [array] $artifactList
 )
 
 . (Join-Path $PSScriptRoot common.ps1)
@@ -132,6 +137,38 @@ if (-not (Test-Path -Path $outDirectory))
   New-Item -ItemType Directory -Force -Path $outDirectory | Out-Null
 }
 
+if ($artifactList)
+{
+  # Filter out null, empty, or whitespace-only entries
+  $filteredArtifacts = @($artifactList | Where-Object { -not [string]::IsNullOrWhiteSpace($_) })
+  
+  if ($filteredArtifacts.Count -eq 0)
+  {
+    Write-Warning "Artifact list contains no valid entries"
+  }
+  else
+  {
+    Write-Host "Filtering package properties to match artifact list: $($filteredArtifacts -join ', ')"
+    $artifactSet = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase)
+    foreach ($artifact in $filteredArtifacts) {
+      $artifactSet.Add($artifact) | Out-Null
+    }
+
+    # Warn about packages missing ArtifactName property
+    $missingArtifactName = $allPackageProperties | Where-Object { $_.PSObject.Properties.Name -notcontains 'ArtifactName' }
+    foreach ($pkg in $missingArtifactName) {
+      Write-Warning "Package '$($pkg.PackageName)' does not have an 'ArtifactName' property and will be excluded from artifact filtering."
+    }
+    $allPackageProperties = $allPackageProperties | Where-Object { $_.ArtifactName -and $artifactSet.Contains($_.ArtifactName) }
+
+    if (!$allPackageProperties)
+    {
+      Write-Error "No packages found matching the provided artifact list"
+      exit 1
+    }
+  }
+}
+
 foreach ($pkg in $allPackageProperties)
 {
   if ($pkg.Name)

eng/common/scripts/stress-testing/find-all-stress-packages.ps1

@@ -31,8 +31,10 @@ function FindStressPackages(
     $filters['stressTest'] = 'true'
     $packages = @()
     $chartFiles = Get-ChildItem -Recurse -Filter 'Chart.yaml' $directory
-    Write-Host "Found chart files:"
-    Write-Host ($chartFiles -join "`n")
+    if ($chartFiles) {
+        Write-Host "Found chart files:"
+        Write-Host ($chartFiles -join "`n")
+    }
 
     if (!$MatrixFileName) {
         $MatrixFileName = 'scenarios-matrix.yaml'

eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1

@@ -172,6 +172,10 @@ function DeployStressTests(
                 -MatrixFilters $MatrixFilters `
                 -MatrixReplace $MatrixReplace `
                 -MatrixNonSparseParameters $MatrixNonSparseParameters)
+    if (!$pkgs -or !$pkgs.Length) {
+        Write-Warning "No stress test packages found in $searchDirectory"
+        exit 0
+    }
     Write-Host "" "Found $($pkgs.Length) stress test packages:"
     Write-Host $pkgs.Directory ""
     foreach ($pkg in $pkgs) {