Skip to content

Commit c1b4775

Browse files
CopilotT-Gro
Copilot
and
T-Gro
committed
Convert regression testing to Azure DevOps template and integrate with PR pipeline
Co-authored-by: T-Gro <[email protected]>
1 parent a19160d commit c1b4775

File tree

3 files changed

+280
-240
lines changed

3 files changed

+280
-240
lines changed

‎azure-pipelines-PR.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -738,6 +738,32 @@ stages:
738738
env:
739739
FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag)
740740
displayName: End to end build tests
741+
742+
# Publish artifacts for regression testing
743+
- task: PublishPipelineArtifact@1
744+
displayName: Publish F# Compiler Artifacts for Regression Tests
745+
inputs:
746+
targetPath: '$(Build.SourcesDirectory)/artifacts'
747+
artifactName: 'FSharpCompilerArtifacts'
748+
publishLocation: pipeline
749+
condition: succeeded()
750+
751+
- task: PublishPipelineArtifact@1
752+
displayName: Publish UseLocalCompiler props file for Regression Tests
753+
inputs:
754+
targetPath: '$(Build.SourcesDirectory)/UseLocalCompiler.Directory.Build.props'
755+
artifactName: 'UseLocalCompilerProps'
756+
publishLocation: pipeline
757+
condition: succeeded()
758+
759+
# F# Compiler Regression Tests using third-party libraries
760+
- template: /eng/templates/regression-test-jobs.yml
761+
parameters:
762+
testMatrix:
763+
- repo: fsprojects/FSharpPlus
764+
commit: f614035b75922aba41ed6a36c2fc986a2171d2b8
765+
buildScript: build.cmd
766+
displayName: FSharpPlus
741767

742768
# Up-to-date - disabled due to it being flaky
743769
#- job: UpToDate_Windows

‎azure-pipelines-regression-test.yml‎

Lines changed: 18 additions & 240 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
1-
# Regression Testing Pipeline
2-
# Tests F# compiler regressions by building third-party F# projects with the freshly built compiler
1+
# DEPRECATED: F# Compiler Regression Testing Pipeline
2+
# This pipeline has been integrated into azure-pipelines-PR.yml
3+
# The regression test logic is now available as a template at eng/templates/regression-test-jobs.yml
4+
# and is automatically run as part of PR builds.
5+
#
6+
# This file is kept for documentation purposes and can be used as a standalone
7+
# regression test pipeline if needed in the future.
38

49
trigger:
510
branches:
@@ -41,25 +46,13 @@ variables:
4146
# Pick up pool provider name behavior from shared yaml template
4247
- template: /eng/common/templates/variables/pool-providers.yml
4348

44-
# Matrix of third-party repositories to test against
45-
# This configuration can be extended in the future
46-
parameters:
47-
- name: testMatrix
48-
type: object
49-
default:
50-
- repo: fsprojects/FSharpPlus
51-
commit: f614035b75922aba41ed6a36c2fc986a2171d2b8
52-
buildScript: build.cmd
53-
displayName: FSharpPlus
54-
5549
stages:
5650
- stage: RegressionTest
57-
displayName: F# Compiler Regression Tests
51+
displayName: F# Compiler Regression Tests (Standalone)
5852
jobs:
59-
# Note: This pipeline should be integrated with azure-pipelines-PR.yml EndToEndBuildTests job
60-
# For now, keeping a minimal build job until architectural integration is completed
53+
# Build the compiler to generate artifacts needed for regression testing
6154
- job: BuildCompiler
62-
displayName: Build F# Compiler (temporary - should use EndToEndBuildTests)
55+
displayName: Build F# Compiler for Regression Tests
6356
pool:
6457
name: $(DncEngPublicBuildPool)
6558
demands: ImageOverride -equals $(WindowsMachineQueueName)
@@ -102,226 +95,11 @@ stages:
10295
publishLocation: pipeline
10396
condition: succeeded()
10497

105-
# Test against third-party repositories
106-
- ${{ each item in parameters.testMatrix }}:
107-
- job: Test_${{ replace(item.repo, '/', '_') }}
108-
displayName: 'Regression Test: ${{ item.displayName }}'
109-
dependsOn: BuildCompiler
110-
pool:
111-
name: $(DncEngPublicBuildPool)
112-
demands: ImageOverride -equals $(WindowsMachineQueueName)
113-
timeoutInMinutes: 60
114-
variables:
115-
TestRepoName: ${{ item.repo }}
116-
TestCommit: ${{ item.commit }}
117-
BuildScript: ${{ item.buildScript }}
118-
DisplayName: ${{ item.displayName }}
119-
steps:
120-
- checkout: none
121-
displayName: Skip default checkout
122-
123-
# Download the F# compiler artifacts
124-
- task: DownloadPipelineArtifact@2
125-
displayName: Download F# Compiler Artifacts
126-
inputs:
127-
artifactName: 'FSharpCompilerArtifacts'
128-
downloadPath: '$(Pipeline.Workspace)/FSharpCompiler'
129-
130-
- task: DownloadPipelineArtifact@2
131-
displayName: Download UseLocalCompiler props
132-
inputs:
133-
artifactName: 'UseLocalCompilerProps'
134-
downloadPath: '$(Pipeline.Workspace)/Props'
135-
136-
# Checkout the third-party repository at specific commit
137-
- task: PowerShell@2
138-
displayName: 'Checkout $(DisplayName) at specific commit'
139-
inputs:
140-
script: |
141-
Write-Host "Cloning repository: $(TestRepoName)"
142-
git clone https://github.com/$(TestRepoName).git $(Pipeline.Workspace)/TestRepo
143-
Set-Location $(Pipeline.Workspace)/TestRepo
144-
145-
Write-Host "Checking out commit: $(TestCommit)"
146-
git checkout $(TestCommit)
147-
148-
Write-Host "Successfully checked out $(TestRepoName) at commit $(TestCommit)"
149-
git log -1 --oneline
150-
151-
Write-Host "Repository structure:"
152-
Get-ChildItem -Name
153-
154-
Write-Host "Verifying build script exists: $(BuildScript)"
155-
if (Test-Path "$(BuildScript)") {
156-
Write-Host "✓ Build script found: $(BuildScript)"
157-
} else {
158-
Write-Host "✗ Build script not found: $(BuildScript)"
159-
Write-Host "Available files in root:"
160-
Get-ChildItem
161-
exit 1
162-
}
163-
164-
# Install appropriate .NET SDK version using global.json if present
165-
- task: UseDotNet@2
166-
displayName: 'Install .NET SDK for $(DisplayName)'
167-
inputs:
168-
packageType: sdk
169-
useGlobalJson: true
170-
workingDirectory: $(Pipeline.Workspace)/TestRepo
171-
installationPath: $(Pipeline.Workspace)/TestRepo/.dotnet
172-
173-
# Setup Directory.Build.props to import UseLocalCompiler configuration
174-
- task: PowerShell@2
175-
displayName: 'Setup local compiler configuration for $(DisplayName)'
176-
inputs:
177-
script: |
178-
Set-Location $(Pipeline.Workspace)/TestRepo
179-
180-
# Create F# script to handle Directory.Build.props setup
181-
$fsharpScript = @'
182-
#r "nuget: System.Xml.ReaderWriter"
183-
open System.IO
184-
open System.Xml
185-
186-
let useLocalCompilerImport = """<Import Project="$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props" />"""
187-
188-
let directoryBuildPropsPath = "Directory.Build.props"
189-
190-
if File.Exists(directoryBuildPropsPath) then
191-
printfn "Directory.Build.props exists, modifying it"
192-
let doc = XmlDocument()
193-
doc.Load(directoryBuildPropsPath)
194-
195-
// Find the Project element
196-
let projectElement = doc.SelectSingleNode("/Project")
197-
if projectElement <> null then
198-
// Check if our import already exists
199-
let existingImport = doc.SelectSingleNode(sprintf "//Import[@Project='$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props']")
200-
if existingImport = null then
201-
let importElement = doc.CreateElement("Import")
202-
importElement.SetAttribute("Project", "$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props")
203-
projectElement.InsertBefore(importElement, projectElement.FirstChild) |> ignore
204-
doc.Save(directoryBuildPropsPath)
205-
printfn "Added UseLocalCompiler import to existing Directory.Build.props"
206-
else
207-
printfn "UseLocalCompiler import already exists"
208-
else
209-
printfn "Warning: Could not find Project element in Directory.Build.props"
210-
else
211-
printfn "Creating new Directory.Build.props"
212-
let content = sprintf "<Project>\n %s\n</Project>" useLocalCompilerImport
213-
File.WriteAllText(directoryBuildPropsPath, content)
214-
215-
printfn "Directory.Build.props content:"
216-
File.ReadAllText(directoryBuildPropsPath) |> printfn "%s"
217-
'@
218-
219-
$fsharpScript | Out-File -FilePath "PrepareRepoForTesting.fsx" -Encoding UTF8
220-
221-
# Run the F# script using dotnet fsi
222-
dotnet fsi PrepareRepoForTesting.fsx
223-
224-
Write-Host "UseLocalCompiler.Directory.Build.props will be referenced from: $(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props"
225-
226-
# Report dotnet info in test environment
227-
- task: PowerShell@2
228-
displayName: 'Report build environment for $(DisplayName)'
229-
inputs:
230-
script: |
231-
Set-Location $(Pipeline.Workspace)/TestRepo
232-
Write-Host "==========================================="
233-
Write-Host "Environment Information for $(DisplayName)"
234-
Write-Host "==========================================="
235-
dotnet --info
236-
Write-Host ""
237-
Write-Host "MSBuild version:"
238-
dotnet msbuild -version
239-
Write-Host ""
240-
Write-Host "F# Compiler artifacts available:"
241-
Get-ChildItem "$(Pipeline.Workspace)\FSharpCompiler\bin\fsc\Release\net9.0" -Name
242-
Write-Host ""
243-
Write-Host "F# Core available:"
244-
if (Test-Path "$(Pipeline.Workspace)\FSharpCompiler\bin\FSharp.Core\Release\netstandard2.0\FSharp.Core.dll") {
245-
Write-Host "✓ FSharp.Core.dll found"
246-
} else {
247-
Write-Host "✗ FSharp.Core.dll not found"
248-
}
249-
Write-Host ""
250-
Write-Host "Directory.Build.props content:"
251-
Get-Content "Directory.Build.props"
252-
Write-Host ""
253-
Write-Host "==========================================="
254-
255-
# Build the third-party project using local F# compiler
256-
- task: PowerShell@2
257-
displayName: 'Build $(DisplayName) with local F# compiler'
258-
env:
259-
# Set environment variables to use local compiler and enforce binary logs
260-
LocalFSharpCompilerPath: $(Pipeline.Workspace)/FSharpCompiler
261-
LoadLocalFSharpBuild: true
262-
LocalFSharpCompilerConfiguration: Release
263-
# Force MSBuild binary logs
264-
MSBUILDBINARYLOGGERENABLED: true
265-
MSBUILDBINARYLOGGER: "*.binlog"
266-
timeoutInMinutes: 45
267-
inputs:
268-
script: |
269-
Set-Location $(Pipeline.Workspace)/TestRepo
270-
Write-Host "============================================"
271-
Write-Host "Starting build for $(DisplayName)"
272-
Write-Host "Repository: $(TestRepoName)"
273-
Write-Host "Commit: $(TestCommit)"
274-
Write-Host "Build Script: $(BuildScript)"
275-
Write-Host "============================================"
276-
Write-Host ""
277-
278-
Write-Host "Executing: $(BuildScript)"
279-
cmd /c "$(BuildScript)"
280-
$exitCode = $LASTEXITCODE
281-
282-
Write-Host ""
283-
Write-Host "============================================"
284-
Write-Host "Build completed for $(DisplayName)"
285-
Write-Host "Exit code: $exitCode"
286-
Write-Host "============================================"
287-
288-
if ($exitCode -ne 0) {
289-
exit $exitCode
290-
}
291-
292-
# Publish only MSBuild binary logs for efficient storage
293-
- task: PublishPipelineArtifact@1
294-
displayName: 'Publish $(DisplayName) Binary Logs'
295-
inputs:
296-
targetPath: '$(Pipeline.Workspace)/TestRepo'
297-
artifactName: '$(DisplayName)_BinaryLogs'
298-
publishLocation: pipeline
299-
artifact: '*.binlog'
300-
condition: always()
301-
continueOnError: true
302-
303-
# Report success/failure
304-
- task: PowerShell@2
305-
displayName: 'Report $(DisplayName) test result'
306-
condition: always()
307-
inputs:
308-
script: |
309-
Set-Location $(Pipeline.Workspace)/TestRepo
310-
Write-Host ""
311-
Write-Host "============================================"
312-
Write-Host "Regression test completed for $(DisplayName)"
313-
Write-Host "Repository: $(TestRepoName)"
314-
Write-Host "Commit: $(TestCommit)"
315-
Write-Host "Build Script: $(BuildScript)"
316-
if ($env:AGENT_JOBSTATUS -eq "Succeeded") {
317-
Write-Host "Status: ✓ SUCCESS"
318-
Write-Host "The $(DisplayName) library builds successfully with the new F# compiler"
319-
} else {
320-
Write-Host "Status: ✗ FAILED"
321-
Write-Host "The $(DisplayName) library failed to build with the new F# compiler"
322-
Write-Host "Check the build logs and artifacts for details"
323-
}
324-
Write-Host "============================================"
325-
326-
Write-Host "Binary logs found:"
327-
Get-ChildItem "*.binlog" -ErrorAction SilentlyContinue | ForEach-Object { Write-Host $_.Name }
98+
# Use the regression test template
99+
- template: /eng/templates/regression-test-jobs.yml
100+
parameters:
101+
testMatrix:
102+
- repo: fsprojects/FSharpPlus
103+
commit: f614035b75922aba41ed6a36c2fc986a2171d2b8
104+
buildScript: build.cmd
105+
displayName: FSharpPlus

0 commit comments

Comments
 (0)