From c4ac84936913dd3210a7e7ab0eb02c3b68cbcf91 Mon Sep 17 00:00:00 2001
From: Juan Ospina <70209456+jeo02@users.noreply.github.com>
Date: Tue, 18 Nov 2025 14:08:52 -0800
Subject: [PATCH 1/2] eval pipeline

---
 eng/common/pipelines/ai-eval-tests.yml | 85 ++++++++++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 eng/common/pipelines/ai-eval-tests.yml

diff --git a/eng/common/pipelines/ai-eval-tests.yml b/eng/common/pipelines/ai-eval-tests.yml
new file mode 100644
index 000000000000..5baf77d52309
--- /dev/null
+++ b/eng/common/pipelines/ai-eval-tests.yml
@@ -0,0 +1,85 @@
+trigger: none
+
+pr:
+  branches:
+    include:
+      - main
+  paths:
+    include:
+      - .github/copilot-instructions.md
+
+parameters:
+  - name: EvalProject
+    type: string
+    default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations'
+  - name: OpenAIEndPoint
+    type: string
+    default: 'https://ai-prmarottai3149546654251245.openai.azure.com/'
+  - name: Model
+    type: string
+    default: 'gpt-5'
+  - name: RepoName
+    type: string
+    default: $(Build.Repository.Name)
+  - name: RepoOwner
+    type: string
+    default: 'Azure'
+  - name: RepoCommit
+    type: string
+    default: $(System.PullRequest.SourceCommitId)
+  - name: WorkingDirectory
+    type: string
+    default: $(System.DefaultWorkingDirectory)
+
+jobs:
+  - job: Run_Eval
+    displayName: 'Run AI Eval'
+    pool:
+      name: $(LINUXPOOL)
+      image: $(LINUXVMIMAGE)
+      os: linux
+
+    steps:
+    - checkout: self
+    - template: /eng/pipelines/templates/steps/install-dotnet.yml
+    - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+      parameters:
+        SkipCheckoutNone: true
+        Repositories:
+          - Name: ${{ parameters.RepoOwner }}/${{ parameters.RepoName }}
+            Commitish: ${{ parameters.RepoCommit }}
+            WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.RepoName }}
+        Paths:
+          - '.github'
+          - '.vscode/**'
+          - 'eng/common/**'
+    
+    - task: AzureCLI@2
+      displayName: 'Run eval'
+      condition: and(succeeded(), ne(variables['Skip.Eval'], 'true'))
+      inputs:
+        azureSubscription: opensource-api-connection
+        scriptType: 'bash'
+        scriptLocation: 'inlineScript'
+        workingDirectory: '${{ parameters.EvalProject }}'
+        inlineScript: |
+          echo "Logged in to Azure"
+          echo "Running eval in project ${{ parameters.EvalProject }}"
+          dotnet test /p:ArtifactsPackagesDir=$(Build.ArtifactStagingDirectory) $(Warn) --logger trx
+      env:
+        DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+        DOTNET_CLI_TELEMETRY_OPTOUT: 1
+        DOTNET_MULTILEVEL_LOOKUP: 0
+        AZURE_OPENAI_MODEL_DEPLOYMENT_NAME: ${{ parameters.Model }}
+        AZURE_OPENAI_ENDPOINT: ${{ parameters.OpenAIEndPoint }}
+        COPILOT_INSTRUCTIONS_REPOSITORY_NAME: ${{ parameters.RepoName }}
+        COPILOT_INSTRUCTIONS_REPOSITORY_OWNER: ${{ parameters.RepoOwner }}
+        COPILOT_INSTRUCTIONS_PATH: $(System.DefaultWorkingDirectory)/${{ parameters.RepoName }}/.github/copilot-instructions.md
+
+    - task: PublishTestResults@2
+      condition: succeededOrFailed()
+      inputs:
+        testResultsFiles: '**/*.trx'
+        testRunTitle: $(System.JobDisplayName)
+        testResultsFormat: 'VSTest'
+        mergeTestResults: true

From ff0e97cb296ca07e3fe80401866c8e94b5fa7f75 Mon Sep 17 00:00:00 2001
From: Juan Ospina <70209456+jeo02@users.noreply.github.com>
Date: Tue, 18 Nov 2025 14:40:05 -0800
Subject: [PATCH 2/2] dotnet

---
 eng/common/pipelines/ai-eval-tests.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/eng/common/pipelines/ai-eval-tests.yml b/eng/common/pipelines/ai-eval-tests.yml
index 5baf77d52309..2bc23db20efa 100644
--- a/eng/common/pipelines/ai-eval-tests.yml
+++ b/eng/common/pipelines/ai-eval-tests.yml
@@ -41,7 +41,11 @@ jobs:
 
     steps:
     - checkout: self
-    - template: /eng/pipelines/templates/steps/install-dotnet.yml
+    - task: UseDotNet@2
+      displayName: 'Install .NET SDK'
+      inputs:
+        version: '8.x'
+        packageType: sdk
     - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
       parameters:
         SkipCheckoutNone: true