test evals

eng/common/pipelines/ai-eval-tests.yml

@@ -0,0 +1,97 @@
+trigger: none
+
+pr:
+  branches:
+    include:
+      - main
+  paths:
+    include:
+      - .github/copilot-instructions.md
+      - eng/common/instructions/azsdk-tools/**
+
+parameters:
+  - name: EvalProject
+    type: string
+    default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations'
+  - name: OpenAIEndPoint
+    type: string
+    default: 'https://ai-prmarottai3149546654251245.openai.azure.com/'
+  - name: Model
+    type: string
+    default: 'gpt-5'
+  - name: ToolsRepoName
+    type: string
+    default: 'azure-sdk-tools'
+  - name: ToolsRepoOwner
+    type: string
+    default: 'Azure'
+
+jobs:
+  - job: Run_Eval
+    variables:
+      - template: /eng/pipelines/templates/variables/globals.yml
+      - template: /eng/pipelines/templates/variables/image.yml
+    displayName: 'Run AI Eval'
+    pool:
+      name: $(LINUXPOOL)
+      image: $(LINUXVMIMAGE)
+      os: linux
+
+    steps:
+    - task: UseDotNet@2
+      displayName: "Use .NET SDK 9.0.x"
+      retryCountOnTaskFailure: 3
+      inputs:
+        packageType: sdk
+        version: 9.0.x
+        performMultiLevelLookup: true
+
+    - task: UseDotNet@2
+      displayName: "Use .NET SDK 8.0.x"
+      retryCountOnTaskFailure: 3
+      inputs:
+        packageType: sdk
+        version: 8.0.x
+        performMultiLevelLookup: true
+
+    - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+      parameters:
+        SkipCheckoutNone: true
+        Repositories:
+          - Name: $(Build.Repository.Name)
+            Commitish: $(Build.SourceVersion)
+            WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)
+          - Name: ${{ parameters.ToolsRepoOwner }}/${{ parameters.ToolsRepoName }}
+            WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.ToolsRepoName }}
+        Paths:
+          - 'tools/**'
+          - eng/common/**
+          - .github/copilot-instructions.md
+
+    - task: AzureCLI@2
+      displayName: 'Run eval'
+      inputs:
+        azureSubscription: opensource-api-connection
+        scriptType: 'bash'
+        scriptLocation: 'inlineScript'
+        workingDirectory: '${{ parameters.ToolsRepoName }}/${{ parameters.EvalProject }}'
+        inlineScript: |
+          echo "Logged in to Azure"
+          echo "Running eval in project ${{ parameters.ToolsRepoName }}/${{ parameters.EvalProject }}"
+          dotnet test /p:ArtifactsPackagesDir=$(Build.ArtifactStagingDirectory) --logger trx
+      env:
+        DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+        DOTNET_CLI_TELEMETRY_OPTOUT: 1
+        DOTNET_MULTILEVEL_LOOKUP: 0
+        AZURE_OPENAI_MODEL_DEPLOYMENT_NAME: ${{ parameters.Model }}
+        AZURE_OPENAI_ENDPOINT: ${{ parameters.OpenAIEndPoint }}
+        REPOSITORY_NAME: $(Build.Repository.Name)
+        COPILOT_INSTRUCTIONS_PATH_MCP_EVALS: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/.github/copilot-instructions.md
+
+    - task: PublishTestResults@2
+      condition: succeededOrFailed()
+      inputs:
+        testResultsFiles: '**/*.trx'
+        testRunTitle: $(System.JobDisplayName)
+        testResultsFormat: 'VSTest'
+        mergeTestResults: true