Skip to content

Commit 5ddb394

Browse files
azure-sdkjeo02Copilot
authored
Sync eng/common directory with azure-sdk-tools for PR 13142 (#2069)
* quotes * generalize * cant use trigger as a job * Update eng/common/pipelines/templates/jobs/ai-eval-job.yml Co-authored-by: Copilot <[email protected]> * target repo name fix * compile time expression * compile time expression * show params for pipeline * fix the pipeline variables * another fix * fix variables --------- Co-authored-by: Juan Ospina <[email protected]> Co-authored-by: Copilot <[email protected]>
1 parent bd72a45 commit 5ddb394

File tree

2 files changed

+124
-73
lines changed

2 files changed

+124
-73
lines changed

‎eng/common/pipelines/ai-evals-tests.yml‎

Lines changed: 24 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -15,83 +15,34 @@ parameters:
1515
default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations'
1616
- name: OpenAIEndPoint
1717
type: string
18-
default: 'https://ai-prmarottai3149546654251245.openai.azure.com/'
18+
default: 'https://openai-shared.openai.azure.com/'
1919
- name: Model
2020
type: string
2121
default: 'gpt-5'
22-
- name: ToolsRepoName
22+
- name: EvalRepoOwner
23+
type: string
24+
default: 'Azure'
25+
- name: EvalRepoName
2326
type: string
2427
default: 'azure-sdk-tools'
25-
- name: ToolsRepoOwner
28+
- name: EvalRepoCommit
2629
type: string
27-
default: 'Azure'
28-
29-
jobs:
30-
- job: Run_Eval
31-
variables:
32-
- template: /eng/pipelines/templates/variables/globals.yml
33-
- template: /eng/pipelines/templates/variables/image.yml
34-
displayName: 'Run AI Eval'
35-
pool:
36-
name: $(LINUXPOOL)
37-
image: $(LINUXVMIMAGE)
38-
os: linux
39-
40-
steps:
41-
- task: UseDotNet@2
42-
displayName: "Use .NET SDK 9.0.x"
43-
retryCountOnTaskFailure: 3
44-
inputs:
45-
packageType: sdk
46-
version: 9.0.x
47-
performMultiLevelLookup: true
48-
49-
- task: UseDotNet@2
50-
displayName: "Use .NET SDK 8.0.x"
51-
retryCountOnTaskFailure: 3
52-
inputs:
53-
packageType: sdk
54-
version: 8.0.x
55-
performMultiLevelLookup: true
56-
57-
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
58-
parameters:
59-
SkipCheckoutNone: true
60-
Repositories:
61-
- Name: $(Build.Repository.Name)
62-
Commitish: $(Build.SourceVersion)
63-
WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)
64-
- Name: ${{ parameters.ToolsRepoOwner }}/${{ parameters.ToolsRepoName }}
65-
WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.ToolsRepoName }}
66-
Paths:
67-
- 'tools/**'
68-
- eng/common/**
69-
- .github/copilot-instructions.md
70-
71-
- task: AzureCLI@2
72-
displayName: 'Run eval'
73-
inputs:
74-
azureSubscription: opensource-api-connection
75-
scriptType: 'bash'
76-
scriptLocation: 'inlineScript'
77-
workingDirectory: '${{ parameters.ToolsRepoName }}/${{ parameters.EvalProject }}'
78-
inlineScript: |
79-
echo "Logged in to Azure"
80-
echo "Running eval in project ${{ parameters.ToolsRepoName }}/${{ parameters.EvalProject }}"
81-
dotnet test /p:ArtifactsPackagesDir=$(Build.ArtifactStagingDirectory) --logger trx
82-
env:
83-
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
84-
DOTNET_CLI_TELEMETRY_OPTOUT: 1
85-
DOTNET_MULTILEVEL_LOOKUP: 0
86-
AZURE_OPENAI_MODEL_DEPLOYMENT_NAME: ${{ parameters.Model }}
87-
AZURE_OPENAI_ENDPOINT: ${{ parameters.OpenAIEndPoint }}
88-
REPOSITORY_NAME: $(Build.Repository.Name)
89-
COPILOT_INSTRUCTIONS_PATH_MCP_EVALS: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/.github/copilot-instructions.md
30+
default: 'main'
9031

91-
- task: PublishTestResults@2
92-
condition: succeededOrFailed()
93-
inputs:
94-
testResultsFiles: '**/*.trx'
95-
testRunTitle: $(System.JobDisplayName)
96-
testResultsFormat: 'VSTest'
97-
mergeTestResults: true
32+
variables:
33+
TargetRepoOwner: ${{ split(variables['Build.Repository.Name'], '/')[0] }}
34+
TargetRepoName: ${{ split(variables['Build.Repository.Name'], '/')[1] }}
35+
TargetRepoCommit: ${{ variables['Build.SourceVersion'] }}
36+
37+
extends:
38+
template: /eng/common/pipelines/templates/jobs/ai-eval-job.yml
39+
parameters:
40+
EvalProject: ${{ parameters.EvalProject }}
41+
OpenAIEndPoint: ${{ parameters.OpenAIEndPoint }}
42+
Model: ${{ parameters.Model }}
43+
EvalRepoOwner: ${{ parameters.EvalRepoOwner }}
44+
EvalRepoName: ${{ parameters.EvalRepoName }}
45+
EvalRepoCommit: ${{ parameters.EvalRepoCommit }}
46+
TargetRepoOwner: $(TargetRepoOwner)
47+
TargetRepoName: $(TargetRepoName)
48+
TargetRepoCommit: $(TargetRepoCommit)

‎eng/common/pipelines/templates/jobs/ai-eval-job.yml‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
parameters:
2+
- name: EvalProject
3+
type: string
4+
default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations'
5+
- name: OpenAIEndPoint
6+
type: string
7+
default: 'https://openai-shared.openai.azure.com/'
8+
- name: Model
9+
type: string
10+
default: 'gpt-5'
11+
# Repository where evals are located
12+
- name: EvalRepoOwner
13+
type: string
14+
default: 'Azure'
15+
- name: EvalRepoName
16+
type: string
17+
default: 'azure-sdk-tools'
18+
- name: EvalRepoCommit
19+
type: string
20+
default: 'main'
21+
# Target repository for copilot instructions
22+
- name: TargetRepoOwner
23+
type: string
24+
- name: TargetRepoName
25+
type: string
26+
- name: TargetRepoCommit
27+
type: string
28+
29+
jobs:
30+
- job: Run_Eval
31+
variables:
32+
- template: /eng/pipelines/templates/variables/globals.yml
33+
- template: /eng/pipelines/templates/variables/image.yml
34+
displayName: 'Run AI Eval'
35+
pool:
36+
name: $(LINUXPOOL)
37+
image: $(LINUXVMIMAGE)
38+
os: linux
39+
40+
steps:
41+
- checkout: none
42+
- task: UseDotNet@2
43+
displayName: "Use .NET SDK 9.0.x"
44+
retryCountOnTaskFailure: 3
45+
inputs:
46+
packageType: sdk
47+
version: 9.0.x
48+
performMultiLevelLookup: true
49+
50+
- task: UseDotNet@2
51+
displayName: "Use .NET SDK 8.0.x"
52+
retryCountOnTaskFailure: 3
53+
inputs:
54+
packageType: sdk
55+
version: 8.0.x
56+
performMultiLevelLookup: true
57+
58+
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
59+
parameters:
60+
SkipCheckoutNone: true
61+
Repositories:
62+
- Name: ${{ parameters.EvalRepoOwner }}/${{ parameters.EvalRepoName }}
63+
Commitish: ${{ parameters.EvalRepoCommit }}
64+
WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.EvalRepoName }}
65+
- Name: ${{ parameters.TargetRepoOwner }}/${{ parameters.TargetRepoName }}
66+
Commitish: ${{ parameters.TargetRepoCommit }}
67+
WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.TargetRepoName }}
68+
Paths:
69+
- 'tools/**'
70+
- 'eng/common/**'
71+
- '.github/copilot-instructions.md'
72+
73+
- task: AzureCLI@2
74+
displayName: 'Run eval'
75+
condition: and(succeeded(), ne(variables['Skip.Eval'], 'true'))
76+
inputs:
77+
azureSubscription: opensource-api-connection
78+
scriptType: 'bash'
79+
scriptLocation: 'inlineScript'
80+
workingDirectory: '$(System.DefaultWorkingDirectory)/${{ parameters.EvalRepoName }}/${{ parameters.EvalProject }}'
81+
inlineScript: |
82+
echo "Logged in to Azure"
83+
echo "Running eval"
84+
dotnet test --logger trx
85+
env:
86+
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
87+
DOTNET_CLI_TELEMETRY_OPTOUT: 1
88+
DOTNET_MULTILEVEL_LOOKUP: 0
89+
AZURE_OPENAI_MODEL_DEPLOYMENT_NAME: ${{ parameters.Model }}
90+
AZURE_OPENAI_ENDPOINT: ${{ parameters.OpenAIEndPoint }}
91+
REPOSITORY_NAME: ${{ parameters.TargetRepoName }}
92+
COPILOT_INSTRUCTIONS_PATH_MCP_EVALS: $(System.DefaultWorkingDirectory)/${{ parameters.TargetRepoName }}/.github/copilot-instructions.md
93+
94+
- task: PublishTestResults@2
95+
condition: succeededOrFailed()
96+
inputs:
97+
testResultsFiles: '**/*.trx'
98+
testRunTitle: $(System.JobDisplayName)
99+
testResultsFormat: 'VSTest'
100+
mergeTestResults: true

0 commit comments

Comments
 (0)