From 0acf0545e8c9fa6c031eae47f953526caca673d9 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Fri, 29 Aug 2025 10:21:29 -0700 Subject: [PATCH 1/5] Prepare for the 1.0.0-preview.7 release of AIShell --- .github/workflows/dotnet.yml | 6 +++--- global.json | 2 +- shell/AIShell.Integration/AIShell.psd1 | 4 ++-- shell/shell.common.props | 2 +- tools/metadata.json | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index 48e82f94..7e62d0e2 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -20,7 +20,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: 8.0.411 + dotnet-version: 8.0.413 - name: Build shell: pwsh run: | @@ -37,7 +37,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: 8.0.411 + dotnet-version: 8.0.413 - name: Build shell: pwsh run: | @@ -54,7 +54,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: 8.0.411 + dotnet-version: 8.0.413 - name: Build shell: pwsh run: | diff --git a/global.json b/global.json index e5f41726..1b22b198 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "8.0.411" + "version": "8.0.413" } } diff --git a/shell/AIShell.Integration/AIShell.psd1 b/shell/AIShell.Integration/AIShell.psd1 index 7b7542e1..5e207637 100644 --- a/shell/AIShell.Integration/AIShell.psd1 +++ b/shell/AIShell.Integration/AIShell.psd1 @@ -1,7 +1,7 @@ @{ RootModule = 'AIShell.psm1' NestedModules = @("AIShell.Integration.dll") - ModuleVersion = '1.0.6' + ModuleVersion = '1.0.7' GUID = 'ECB8BEE0-59B9-4DAE-9D7B-A990B480279A' Author = 'Microsoft Corporation' CompanyName = 'Microsoft Corporation' @@ -14,5 +14,5 @@ VariablesToExport = '*' AliasesToExport = @('aish', 'askai', 'fixit', 'airun') HelpInfoURI = 'https://aka.ms/aishell-help' - PrivateData = @{ PSData = @{ Prerelease = 'preview6'; ProjectUri = 'https://github.com/PowerShell/AIShell' } } + PrivateData = @{ PSData = @{ Prerelease = 'preview7'; ProjectUri = 'https://github.com/PowerShell/AIShell' } } } diff --git a/shell/shell.common.props b/shell/shell.common.props index 823ef2ea..58204c6f 100644 --- a/shell/shell.common.props +++ b/shell/shell.common.props @@ -8,7 +8,7 @@ net8.0 enable 12.0 - 1.0.0-preview.6 + 1.0.0-preview.7 true true diff --git a/tools/metadata.json b/tools/metadata.json index 14857f91..54f4334f 100644 --- a/tools/metadata.json +++ b/tools/metadata.json @@ -1,3 +1,3 @@ { - "AgentsToInclude": "*" + "AgentsToInclude": ["msaz", "openai-gpt"] } From 80f17687e8f661b8d2c0d940e48991948668ef52 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Fri, 29 Aug 2025 12:31:30 -0700 Subject: [PATCH 2/5] Use new LinuxContainerImage and enable binskim globally --- .pipelines/Build-Official.yml | 5 +++-- .pipelines/Package-Official.yml | 5 +++-- .pipelines/Release-Official.yml | 2 +- .pipelines/templates/linux-build.yml | 2 -- .pipelines/templates/linux-package.yml | 2 -- .pipelines/templates/mac-package.yml | 2 -- .pipelines/templates/module-build.yml | 2 -- .pipelines/templates/module-package.yml | 2 -- .pipelines/templates/nupkg-package.yml | 2 -- .pipelines/templates/windows-build.yml | 2 -- .pipelines/templates/windows-package.yml | 2 -- 11 files changed, 7 insertions(+), 21 deletions(-) diff --git a/.pipelines/Build-Official.yml b/.pipelines/Build-Official.yml index 73741184..4cfe6c30 100644 --- a/.pipelines/Build-Official.yml +++ b/.pipelines/Build-Official.yml @@ -28,7 +28,7 @@ variables: - name: BUILDSECMON_OPT_IN value: true - name: LinuxContainerImage - value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 + value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 - name: WindowsContainerImage value: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest - name: CDP_DEFINITION_BUILD_COUNT @@ -68,7 +68,8 @@ extends: enabled: true scanFolder: $(Build.SourcesDirectory) binskim: - enabled: false + enabled: true + exactToolVersion: 4.4.2 apiscan: enabled: false tsaOptionsFile: .config\tsaoptions.json diff --git a/.pipelines/Package-Official.yml b/.pipelines/Package-Official.yml index f08da0b5..11ba2351 100644 --- a/.pipelines/Package-Official.yml +++ b/.pipelines/Package-Official.yml @@ -18,7 +18,7 @@ variables: - name: WindowsContainerImage value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' # Docker image which is used to build the project - name: LinuxContainerImage - value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 + value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 - group: mscodehub-feed-read-general - group: mscodehub-feed-read-akv - name: branchCounterKey @@ -71,7 +71,8 @@ extends: enabled: true scanFolder: $(Build.SourcesDirectory) binskim: - enabled: false + enabled: true + exactToolVersion: 4.4.2 apiscan: enabled: false tsaOptionsFile: .config\tsaoptions.json diff --git a/.pipelines/Release-Official.yml b/.pipelines/Release-Official.yml index 753b13fb..071bb404 100644 --- a/.pipelines/Release-Official.yml +++ b/.pipelines/Release-Official.yml @@ -20,7 +20,7 @@ variables: - name: WindowsContainerImage value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - name: LinuxContainerImage - value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 + value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 resources: repositories: diff --git a/.pipelines/templates/linux-build.yml b/.pipelines/templates/linux-build.yml index 4290cc18..c15e8db6 100644 --- a/.pipelines/templates/linux-build.yml +++ b/.pipelines/templates/linux-build.yml @@ -21,8 +21,6 @@ jobs: value: false - name: ob_sdl_codeSignValidation_enabled value: false - - name: ob_sdl_binskim_enabled - value: true - name: ob_sdl_tsa_configFile value: $(repoRoot)\.config\tsaoptions.json - name: Architecture diff --git a/.pipelines/templates/linux-package.yml b/.pipelines/templates/linux-package.yml index 58ab6aab..5eb9376c 100644 --- a/.pipelines/templates/linux-package.yml +++ b/.pipelines/templates/linux-package.yml @@ -26,8 +26,6 @@ jobs: value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - name: repoRoot value: $(Build.SourcesDirectory)/AIShell - - name: ob_sdl_binskim_enabled - value: true - name: ob_sdl_tsa_configFile value: $(repoRoot)/.config/tsaoptions.json - name: Architecture diff --git a/.pipelines/templates/mac-package.yml b/.pipelines/templates/mac-package.yml index fdee5161..48f1b2cd 100644 --- a/.pipelines/templates/mac-package.yml +++ b/.pipelines/templates/mac-package.yml @@ -29,8 +29,6 @@ jobs: value: false - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: ob_sdl_binskim_enabled - value: true - name: repoRoot value: $(Build.SourcesDirectory) - name: Architecture diff --git a/.pipelines/templates/module-build.yml b/.pipelines/templates/module-build.yml index 48844113..6f2d6c6e 100644 --- a/.pipelines/templates/module-build.yml +++ b/.pipelines/templates/module-build.yml @@ -18,8 +18,6 @@ jobs: value: false - name: ob_sdl_codeSignValidation_enabled value: false - - name: ob_sdl_binskim_enabled - value: true - name: ob_sdl_tsa_configFile value: $(repoRoot)\.config\tsaoptions.json #CodeQL tasks added manually to workaround signing failures diff --git a/.pipelines/templates/module-package.yml b/.pipelines/templates/module-package.yml index 22badf81..b044bb92 100644 --- a/.pipelines/templates/module-package.yml +++ b/.pipelines/templates/module-package.yml @@ -22,8 +22,6 @@ jobs: value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - name: repoRoot value: $(Build.SourcesDirectory)/AIShell - - name: ob_sdl_binskim_enabled - value: true - name: ob_sdl_tsa_configFile value: $(repoRoot)/.config/tsaoptions.json - name: ob_signing_setup_enabled diff --git a/.pipelines/templates/nupkg-package.yml b/.pipelines/templates/nupkg-package.yml index b4ff8db9..d2fd2502 100644 --- a/.pipelines/templates/nupkg-package.yml +++ b/.pipelines/templates/nupkg-package.yml @@ -18,8 +18,6 @@ jobs: value: '$(Build.ArtifactStagingDirectory)\ONEBRANCH_ARTIFACT' - name: repoRoot value: $(Build.SourcesDirectory)\AIShell - - name: ob_sdl_binskim_enabled - value: true - name: ob_sdl_tsa_configFile value: $(repoRoot)\.config\tsaoptions.json - group: mscodehub-feed-read-general diff --git a/.pipelines/templates/windows-build.yml b/.pipelines/templates/windows-build.yml index 09258ba5..3a83b6c1 100644 --- a/.pipelines/templates/windows-build.yml +++ b/.pipelines/templates/windows-build.yml @@ -20,8 +20,6 @@ jobs: value: $(Build.SourcesDirectory)\AIShell - name: ob_sdl_codeSignValidation_enabled value: false - - name: ob_sdl_binskim_enabled - value: true - name: ob_sdl_tsa_configFile value: $(repoRoot)\.config\tsaoptions.json - name: Architecture diff --git a/.pipelines/templates/windows-package.yml b/.pipelines/templates/windows-package.yml index 8bc82090..b06f3cff 100644 --- a/.pipelines/templates/windows-package.yml +++ b/.pipelines/templates/windows-package.yml @@ -27,8 +27,6 @@ jobs: value: '$(Build.ArtifactStagingDirectory)\ONEBRANCH_ARTIFACT' - name: repoRoot value: $(Build.SourcesDirectory)\AIShell - - name: ob_sdl_binskim_enabled - value: true - name: ob_sdl_tsa_configFile value: $(repoRoot)\.config\tsaoptions.json - name: Architecture From 007383b1b5e85310529b7a1724fc298105f05b2f Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Wed, 3 Sep 2025 16:19:11 -0700 Subject: [PATCH 3/5] Add fxdependent build and enable APIScan --- .pipelines/Build-Official.yml | 5 + .pipelines/templates/apiscan.yaml | 188 ++++++++++++++++++ build.psm1 | 10 +- .../AIShell.Interpreter.Agent.csproj | 7 + .../AIShell.Ollama.Agent.csproj | 7 + .../AIShell.OpenAI.Agent.csproj | 7 + .../Microsoft.Azure.Agent.csproj | 7 + shell/shell.common.props | 6 + 8 files changed, 234 insertions(+), 3 deletions(-) create mode 100644 .pipelines/templates/apiscan.yaml diff --git a/.pipelines/Build-Official.yml b/.pipelines/Build-Official.yml index 4cfe6c30..9d957d1e 100644 --- a/.pipelines/Build-Official.yml +++ b/.pipelines/Build-Official.yml @@ -112,3 +112,8 @@ extends: displayName: module - build and sign jobs: - template: /.pipelines/templates/module-build.yml@self + + - stage: APIScan + displayName: 'ApiScan' + jobs: + - template: /.pipelines/templates/apiscan.yaml@self diff --git a/.pipelines/templates/apiscan.yaml b/.pipelines/templates/apiscan.yaml new file mode 100644 index 00000000..f1e5ca33 --- /dev/null +++ b/.pipelines/templates/apiscan.yaml @@ -0,0 +1,188 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +jobs: +- job: APIScan + displayName: APIScan with fxdependent build + condition: succeeded() + pool: + type: windows + variables: + - name: runCodesignValidationInjection + value : false + - name: NugetSecurityAnalysisWarningLevel + value: none + # PAT permissions NOTE: Declare a SymbolServerPAT variable in this group with a 'microsoft' organizanization scoped PAT with 'Symbols' Read permission. + # A PAT in the wrong org will give a single Error 203. No PAT will give a single Error 401, and individual pdbs may be missing even if permissions are correct. + - group: symbols + # Defines the variables APIScanClient, APIScanTenant and APIScanSecret + - group: PS-PS-APIScan + - group: DotNetPrivateBuildAccess + - group: 'Azure Blob variable group' + - group: ReleasePipelineSecrets + - group: mscodehub-feed-read-general + - group: mscodehub-feed-read-akv + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: repoRoot + value: $(Build.SourcesDirectory)\AIShell + - name: ob_sdl_tsa_configFile + value: $(repoRoot)\.config\tsaoptions.json + - name: ob_sdl_apiscan_enabled + value: true + - name: ob_sdl_apiscan_softwareName + value: 'AIShell' + - name: ob_sdl_apiscan_versionNumber + value: '1.0' + - name: ob_sdl_apiscan_isLargeApp + value: false + - name: ob_sdl_apiscan_symbolsFolder + value: $(SymbolsServerUrl);$(ob_outputDirectory) + - name: Codeql.SourceRoot + value: $(repoRoot) + + # APIScan can take a long time + timeoutInMinutes: 180 + + steps: + - checkout: self + clean: true + fetchTags: true + fetchDepth: 1000 + displayName: Checkout AIShell + retryCountOnTaskFailure: 1 + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - template: update-nuget-config.yml@self + parameters: + repoRoot: $(repoRoot) + + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + workingDirectory: $(Build.SourcesDirectory)" + + - pwsh: | + dotnet tool install dotnet-symbol --tool-path $(Agent.ToolsDirectory)\tools\dotnet-symbol + $symbolToolPath = Get-ChildItem -Path $(Agent.ToolsDirectory)\tools\dotnet-symbol\dotnet-symbol.exe | Select-Object -First 1 -ExpandProperty FullName + Write-Host "##vso[task.setvariable variable=symbolToolPath]$symbolToolPath" + displayName: Install dotnet-symbol + workingDirectory: '$(repoRoot)' + retryCountOnTaskFailure: 2 + + - task: AzurePowerShell@5 + displayName: Download winverify-private Artifacts + inputs: + azureSubscription: az-blob-cicd-infra + scriptType: inlineScript + azurePowerShellVersion: LatestVersion + workingDirectory: '$(repoRoot)' + pwsh: true + inline: | + # download smybols for getfilesiginforedist.dll + $storageAccountName = "pscoretestdata" + $containerName = 'winverify-private' + $winverifySymbolsPath = New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)/winverify-symbols' -Force + $dllName = 'getfilesiginforedist.dll' + $winverifySymbolsDllPath = Join-Path $winverifySymbolsPath $dllName + + $context = New-AzStorageContext -StorageAccountName $storageAccountName -UseConnectedAccount + Get-AzStorageBlobContent -Container $containerName -Blob $dllName -Destination $winverifySymbolsDllPath -Context $context + + - pwsh: | + Get-ChildItem -Path '$(System.ArtifactsDirectory)/winverify-symbols' + displayName: Capture winverify-private Artifacts + workingDirectory: '$(repoRoot)' + condition: succeededOrFailed() + + - task: CodeQL3000Init@0 # Add CodeQL Init task right before your 'Build' step. + displayName: 🔏 CodeQL 3000 Init + condition: eq(variables['CODEQL_ENABLED'], 'true') + inputs: + Language: csharp + + - pwsh: | + Import-Module $(repoRoot)/build.psm1 -Force + Start-Build -Configuration StaticAnalysis -Runtime fxdependent -Clean -Verbose + + $outputJson = '$(repoRoot)/_build_output_.json' + if (-not (Test-Path $outputJson)) { + throw "'_build_output_.json' was not produced." + } + + $result = Get-Content $outputJson | ConvertFrom-Json + Write-Verbose "App path: $($result.App)" -Verbose + $OutputFolder = $result.App + + Write-Verbose -Verbose -Message "Deleting ref folder from output folder" + if (Test-Path $OutputFolder/ref) { + Remove-Item -Recurse -Force $OutputFolder/ref + } + + Copy-Item -Path "$OutputFolder\*" -Destination '$(ob_outputDirectory)' -Recurse -Verbose + workingDirectory: '$(repoRoot)' + displayName: 'Build AIShell Source' + + - pwsh: | + # Only key windows runtimes + Get-ChildItem -Path '$(ob_outputDirectory)\runtimes\*' -File -Recurse | Where-Object {$_.FullName -notmatch '.*\/runtimes\/win'} | Foreach-Object { + Write-Verbose -Verbose -Message "Deleting $($_.FullName)" + Remove-Item -Force -Verbose -Path $_.FullName + } + + # Temporarily remove runtimes/win-x86 due to issues with that runtime + Get-ChildItem -Path '$(ob_outputDirectory)\runtimes\*' -File -Recurse | Where-Object {$_.FullName -match '.*\/runtimes\/win-x86\/'} | Foreach-Object { + Write-Verbose -Verbose -Message "Deleting $($_.FullName)" + Remove-Item -Force -Verbose -Path $_.FullName + } + + workingDirectory: '$(repoRoot)' + displayName: 'Remove unused runtimes' + + - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. + displayName: 🔏 CodeQL 3000 Finalize + condition: eq(variables['CODEQL_ENABLED'], 'true') + + - pwsh: | + Get-ChildItem -Path env: | Out-String -width 150 -Stream | write-Verbose -Verbose + workingDirectory: '$(repoRoot)' + displayName: Capture Environment + condition: succeededOrFailed() + + # Explicitly download symbols for the drop since the SDL image doesn't have http://SymWeb access and APIScan cannot handle https yet. + - pwsh: | + $pat = '$(SymbolServerPAT)' + if ($pat -like '*PAT*' -or $pat -eq '') { + throw 'No PAT defined' + } + $url = 'https://microsoft.artifacts.visualstudio.com/defaultcollection/_apis/symbol/symsrv' + $(symbolToolPath) --authenticated-server-path $(SymbolServerPAT) $url --symbols -d "$env:ob_outputDirectory\*" --recurse-subdirectories + displayName: 'Download Symbols for binaries' + retryCountOnTaskFailure: 2 + workingDirectory: '$(repoRoot)' + + - pwsh: | + Get-ChildItem '$(ob_outputDirectory)' -File -Recurse | + Foreach-Object { + [pscustomobject]@{ + Path = $_.FullName + Version = $_.VersionInfo.FileVersion + Md5Hash = (Get-FileHash -Algorithm MD5 -Path $_.FullName).Hash + Sha512Hash = (Get-FileHash -Algorithm SHA512 -Path $_.FullName).Hash + } + } | Export-Csv -Path '$(Build.SourcesDirectory)/ReleaseFileHash.csv' + workingDirectory: '$(repoRoot)' + displayName: 'Create release file hash artifact' + + - pwsh: | + Copy-Item -Path '$(Build.SourcesDirectory)/ReleaseFileHash.csv' -Destination '$(ob_outputDirectory)' -Verbose + displayName: 'Publish Build File Hash artifact' + + - pwsh: | + Get-ChildItem -Path env: | Out-String -width 150 -Stream | write-Verbose -Verbose + displayName: Capture Environment + condition: succeededOrFailed() + workingDirectory: '$(repoRoot)' diff --git a/build.psm1 b/build.psm1 index f84f5de3..cedb657e 100644 --- a/build.psm1 +++ b/build.psm1 @@ -13,11 +13,11 @@ function Start-Build [CmdletBinding()] param ( [Parameter()] - [ValidateSet('Debug', 'Release')] + [ValidateSet('Debug', 'Release', 'StaticAnalysis')] [string] $Configuration = "Debug", [Parameter()] - [ValidateSet('win-x86', 'win-x64', 'win-arm64', 'linux-x64', 'linux-arm64', 'osx-x64', 'osx-arm64')] + [ValidateSet('win-x86', 'win-x64', 'win-arm64', 'linux-x64', 'linux-arm64', 'osx-x64', 'osx-arm64', 'fxdependent')] [string] $Runtime = [NullString]::Value, [Parameter()] @@ -97,7 +97,11 @@ function Start-Build Write-Host "`n[Build AI Shell ...]`n" -ForegroundColor Green $app_csproj = GetProjectFile $app_dir - dotnet publish $app_csproj -c $Configuration -o $app_out_dir -r $RID --sc + if ($RID -eq 'fxdependent') { + dotnet publish $app_csproj -c $Configuration -o $app_out_dir --no-self-contained + } else { + dotnet publish $app_csproj -c $Configuration -o $app_out_dir -r $RID --sc + } ## Move the 'Modules' folder to the appbase folder. if ($LASTEXITCODE -eq 0) { diff --git a/shell/agents/AIShell.Interpreter.Agent/AIShell.Interpreter.Agent.csproj b/shell/agents/AIShell.Interpreter.Agent/AIShell.Interpreter.Agent.csproj index d5e56a51..e8e3285c 100644 --- a/shell/agents/AIShell.Interpreter.Agent/AIShell.Interpreter.Agent.csproj +++ b/shell/agents/AIShell.Interpreter.Agent/AIShell.Interpreter.Agent.csproj @@ -13,6 +13,13 @@ false None + true + + + + true + + full diff --git a/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj b/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj index 3e7464d1..044e7bcc 100644 --- a/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj +++ b/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj @@ -14,6 +14,13 @@ false None + true + + + + true + + full diff --git a/shell/agents/AIShell.OpenAI.Agent/AIShell.OpenAI.Agent.csproj b/shell/agents/AIShell.OpenAI.Agent/AIShell.OpenAI.Agent.csproj index c9d9a0da..abd985ec 100644 --- a/shell/agents/AIShell.OpenAI.Agent/AIShell.OpenAI.Agent.csproj +++ b/shell/agents/AIShell.OpenAI.Agent/AIShell.OpenAI.Agent.csproj @@ -13,6 +13,13 @@ false None + true + + + + true + + full diff --git a/shell/agents/Microsoft.Azure.Agent/Microsoft.Azure.Agent.csproj b/shell/agents/Microsoft.Azure.Agent/Microsoft.Azure.Agent.csproj index d264f12a..98cb3385 100644 --- a/shell/agents/Microsoft.Azure.Agent/Microsoft.Azure.Agent.csproj +++ b/shell/agents/Microsoft.Azure.Agent/Microsoft.Azure.Agent.csproj @@ -13,6 +13,13 @@ false None + true + + + + true + + full diff --git a/shell/shell.common.props b/shell/shell.common.props index 58204c6f..b122203f 100644 --- a/shell/shell.common.props +++ b/shell/shell.common.props @@ -24,4 +24,10 @@ true + + true + + full + + From 6e5563f3a6a6221315bf6cf00abc9187ff4641cd Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Thu, 4 Sep 2025 10:13:05 -0700 Subject: [PATCH 4/5] Use a dedicated APIScan pipeline with 'NonOfficial' pipeline template --- .pipelines/APIScan.yml | 101 ++++++++++++++ .pipelines/Build-Official.yml | 5 - .../apiscan-build.yaml} | 130 ++++++++++-------- 3 files changed, 170 insertions(+), 66 deletions(-) create mode 100644 .pipelines/APIScan.yml rename .pipelines/templates/{apiscan.yaml => compliance/apiscan-build.yaml} (52%) diff --git a/.pipelines/APIScan.yml b/.pipelines/APIScan.yml new file mode 100644 index 00000000..f3039c3e --- /dev/null +++ b/.pipelines/APIScan.yml @@ -0,0 +1,101 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: apiscan-$(BUILD.SOURCEBRANCHNAME)-$(Build.BuildId) +trigger: none + +parameters: + - name: FORCE_CODEQL + displayName: Debugging - Enable CodeQL and set cadence to 1 hour + type: boolean + default: false + - name: SkipVerifyPackages + type: boolean + default: false + +variables: + # PAT permissions NOTE: Declare a SymbolServerPAT variable in this group with a 'microsoft' organizanization scoped PAT with 'Symbols' Read permission. + # A PAT in the wrong org will give a single Error 203. No PAT will give a single Error 401, and individual pdbs may be missing even if permissions are correct. + - group: symbols + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: CDP_DEFINITION_BUILD_COUNT + value: $[counter('', 0)] + # Defines the variables CgPat, CgOrganization, and CgProject + - group: 'ComponentGovernance' + - group: 'PoolNames' + - name: LinuxContainerImage + value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 + - name: WindowsContainerImage + value: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + - ${{ if eq(parameters['FORCE_CODEQL'],'true') }}: + # Cadence is hours before CodeQL will allow a re-upload of the database + - name: CodeQL.Cadence + value: 0 + - name: CODEQL_ENABLED + ${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), eq(parameters['FORCE_CODEQL'],'true')) }}: + value: true + ${{ else }}: + value: false + - name: Codeql.TSAEnabled + value: $(CODEQL_ENABLED) + # AnalyzeInPipeline: false = upload results + # AnalyzeInPipeline: true = do not upload results + - name: Codeql.AnalyzeInPipeline + value: $(CODEQL_ENABLED) + +resources: + repositories: + - repository: templates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + +extends: + template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates + parameters: + featureFlags: + WindowsHostVersion: + Version: 2022 + globalSdl: + codeql: + compiled: + enabled: $(CODEQL_ENABLED) + tsaEnabled: $(CODEQL_ENABLED) # This enables TSA bug filing only for CodeQL 3000 + armory: + enabled: false + sbom: + enabled: false + cg: + enabled: true + ignoreDirectories: 'docs,shell,tools' + tsa: + enabled: true # onebranch publish all SDL results to TSA. If TSA is disabled all SDL tools will forced into 'break' build mode. + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + binskim: + break: true # always break the build on binskim issues in addition to TSA upload + policheck: + break: true # always break the build on policheck issues. You can disable it by setting to 'false' + # APIScan requires a non-Ready-To-Run build + apiscan: + enabled: true + softwareName: 'AIShell' # Default is repo name + versionNumber: '1.0' # Default is build number + isLargeApp: false # Default: false. + symbolsFolder: $(SymbolsServerUrl);$(ob_outputDirectory) + tsaOptionsFile: .config\tsaoptions.json + psscriptanalyzer: + enabled: true + policyName: Microsoft + break: false + + stages: + - stage: APIScan + displayName: 'ApiScan' + dependsOn: [] + jobs: + - template: /.pipelines/templates/compliance/apiscan-build.yaml@self + parameters: + parentJobs: [] diff --git a/.pipelines/Build-Official.yml b/.pipelines/Build-Official.yml index 9d957d1e..4cfe6c30 100644 --- a/.pipelines/Build-Official.yml +++ b/.pipelines/Build-Official.yml @@ -112,8 +112,3 @@ extends: displayName: module - build and sign jobs: - template: /.pipelines/templates/module-build.yml@self - - - stage: APIScan - displayName: 'ApiScan' - jobs: - - template: /.pipelines/templates/apiscan.yaml@self diff --git a/.pipelines/templates/apiscan.yaml b/.pipelines/templates/compliance/apiscan-build.yaml similarity index 52% rename from .pipelines/templates/apiscan.yaml rename to .pipelines/templates/compliance/apiscan-build.yaml index f1e5ca33..39625615 100644 --- a/.pipelines/templates/apiscan.yaml +++ b/.pipelines/templates/compliance/apiscan-build.yaml @@ -12,32 +12,17 @@ jobs: value : false - name: NugetSecurityAnalysisWarningLevel value: none - # PAT permissions NOTE: Declare a SymbolServerPAT variable in this group with a 'microsoft' organizanization scoped PAT with 'Symbols' Read permission. - # A PAT in the wrong org will give a single Error 203. No PAT will give a single Error 401, and individual pdbs may be missing even if permissions are correct. - - group: symbols # Defines the variables APIScanClient, APIScanTenant and APIScanSecret - group: PS-PS-APIScan - - group: DotNetPrivateBuildAccess - group: 'Azure Blob variable group' - - group: ReleasePipelineSecrets - group: mscodehub-feed-read-general - group: mscodehub-feed-read-akv - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + value: '$(Build.ArtifactStagingDirectory)\ONEBRANCH_ARTIFACT' - name: repoRoot value: $(Build.SourcesDirectory)\AIShell - name: ob_sdl_tsa_configFile value: $(repoRoot)\.config\tsaoptions.json - - name: ob_sdl_apiscan_enabled - value: true - - name: ob_sdl_apiscan_softwareName - value: 'AIShell' - - name: ob_sdl_apiscan_versionNumber - value: '1.0' - - name: ob_sdl_apiscan_isLargeApp - value: false - - name: ob_sdl_apiscan_symbolsFolder - value: $(SymbolsServerUrl);$(ob_outputDirectory) - name: Codeql.SourceRoot value: $(repoRoot) @@ -54,7 +39,7 @@ jobs: env: ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - - template: update-nuget-config.yml@self + - template: ..\update-nuget-config.yml@self parameters: repoRoot: $(repoRoot) @@ -65,13 +50,13 @@ jobs: packageType: 'sdk' workingDirectory: $(Build.SourcesDirectory)" - - pwsh: | - dotnet tool install dotnet-symbol --tool-path $(Agent.ToolsDirectory)\tools\dotnet-symbol - $symbolToolPath = Get-ChildItem -Path $(Agent.ToolsDirectory)\tools\dotnet-symbol\dotnet-symbol.exe | Select-Object -First 1 -ExpandProperty FullName - Write-Host "##vso[task.setvariable variable=symbolToolPath]$symbolToolPath" - displayName: Install dotnet-symbol - workingDirectory: '$(repoRoot)' - retryCountOnTaskFailure: 2 + # - pwsh: | + # dotnet tool install dotnet-symbol --tool-path $(Agent.ToolsDirectory)\tools\dotnet-symbol + # $symbolToolPath = Get-ChildItem -Path $(Agent.ToolsDirectory)\tools\dotnet-symbol\dotnet-symbol.exe | Select-Object -First 1 -ExpandProperty FullName + # Write-Host "##vso[task.setvariable variable=symbolToolPath]$symbolToolPath" + # displayName: Install dotnet-symbol + # workingDirectory: '$(repoRoot)' + # retryCountOnTaskFailure: 2 - task: AzurePowerShell@5 displayName: Download winverify-private Artifacts @@ -85,7 +70,7 @@ jobs: # download smybols for getfilesiginforedist.dll $storageAccountName = "pscoretestdata" $containerName = 'winverify-private' - $winverifySymbolsPath = New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)/winverify-symbols' -Force + $winverifySymbolsPath = New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)\winverify-symbols' -Force $dllName = 'getfilesiginforedist.dll' $winverifySymbolsDllPath = Join-Path $winverifySymbolsPath $dllName @@ -93,7 +78,7 @@ jobs: Get-AzStorageBlobContent -Container $containerName -Blob $dllName -Destination $winverifySymbolsDllPath -Context $context - pwsh: | - Get-ChildItem -Path '$(System.ArtifactsDirectory)/winverify-symbols' + Get-ChildItem -Path '$(System.ArtifactsDirectory)\winverify-symbols' displayName: Capture winverify-private Artifacts workingDirectory: '$(repoRoot)' condition: succeededOrFailed() @@ -105,42 +90,70 @@ jobs: Language: csharp - pwsh: | - Import-Module $(repoRoot)/build.psm1 -Force + Import-Module '$(repoRoot)\build.psm1' -Force Start-Build -Configuration StaticAnalysis -Runtime fxdependent -Clean -Verbose - $outputJson = '$(repoRoot)/_build_output_.json' + $outputJson = '$(repoRoot)\_build_output_.json' if (-not (Test-Path $outputJson)) { throw "'_build_output_.json' was not produced." } $result = Get-Content $outputJson | ConvertFrom-Json - Write-Verbose "App path: $($result.App)" -Verbose $OutputFolder = $result.App + Write-Verbose "App path: $OutputFolder" -Verbose + + Write-Verbose -Verbose -Message "Deleting the 'ref' folder ..." + if (Test-Path $OutputFolder\ref) { + Remove-Item -Recurse -Force $OutputFolder\ref + } - Write-Verbose -Verbose -Message "Deleting ref folder from output folder" - if (Test-Path $OutputFolder/ref) { - Remove-Item -Recurse -Force $OutputFolder/ref + # Only keep windows runtimes + Write-Verbose -Verbose -Message "Deleting non-win-x64 runtimes ..." + Get-ChildItem -Path "$OutputFolder\runtimes\*" | Where-Object {$_.FullName -notmatch '.*\\runtimes\\win'} | Foreach-Object { + Write-Verbose -Verbose -Message "Deleting $($_.FullName)" + Remove-Item -Path $_.FullName -Recurse -Force } - Copy-Item -Path "$OutputFolder\*" -Destination '$(ob_outputDirectory)' -Recurse -Verbose + # Remove win-x86/arm/arm64 runtimes due to issues with those runtimes + Write-Verbose -Verbose -Message "Temporarily deleting win-x86/arm/arm64 runtimes ..." + Get-ChildItem -Path "$OutputFolder\runtimes\*" | Where-Object {$_.FullName -match '.*\\runtimes\\win-(x86|arm)'} | Foreach-Object { + Write-Verbose -Verbose -Message "Deleting $($_.FullName)" + Remove-Item -Path $_.FullName -Recurse -Force + } + + Write-Host + Write-Verbose -Verbose -Message "Show content in 'runtimes' folder:" + Get-ChildItem -Path "$OutputFolder\runtimes" + Write-Host + + # Replace 'getfilesiginforedist.dll' from the 'runtimes\win-x64' folder + Write-Verbose -Verbose -Message "Replace 'getfilesiginforedist.dll':" + $targetFile = Get-ChildItem -Path "$OutputFolder\runtimes\*" -Recurse | Where-Object {$_.Name -eq 'getfilesiginforedist.dll'} + Remove-Item -Path $targetFile.FullName -Verbose + Copy-Item -Path '$(System.ArtifactsDirectory)\winverify-symbols\getfilesiginforedist.dll' -Destination $targetFile.FullName -Verbose + + Write-Host "##vso[task.setvariable variable=appPath]$OutputFolder" workingDirectory: '$(repoRoot)' displayName: 'Build AIShell Source' - pwsh: | - # Only key windows runtimes - Get-ChildItem -Path '$(ob_outputDirectory)\runtimes\*' -File -Recurse | Where-Object {$_.FullName -notmatch '.*\/runtimes\/win'} | Foreach-Object { - Write-Verbose -Verbose -Message "Deleting $($_.FullName)" - Remove-Item -Force -Verbose -Path $_.FullName + $Destination = '$(ob_outputDirectory)' + if (-not (Test-Path $Destination)) { + Write-Verbose -Verbose -Message "Creating destination folder '$Destination'" + $null = mkdir $Destination } - # Temporarily remove runtimes/win-x86 due to issues with that runtime - Get-ChildItem -Path '$(ob_outputDirectory)\runtimes\*' -File -Recurse | Where-Object {$_.FullName -match '.*\/runtimes\/win-x86\/'} | Foreach-Object { - Write-Verbose -Verbose -Message "Deleting $($_.FullName)" - Remove-Item -Force -Verbose -Path $_.FullName - } + Write-Verbose -Verbose -Message "Copy AIShell app to '$Destination':" + Copy-Item -Path '$(appPath)\*' -Destination $Destination -Recurse -Verbose + Write-Host + Write-Verbose -Verbose -Message "Show content in '$Destination':" + Get-ChildItem -Path $Destination | Out-String -width 150 + Write-Host + Write-Verbose -Verbose -Message "Show content in '$Destination\runtimes':" + Get-ChildItem -Path "$Destination\runtimes" -Recurse | Out-String -width 150 workingDirectory: '$(repoRoot)' - displayName: 'Remove unused runtimes' + displayName: 'Copy AIShell to ob_outputDirectory' - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. displayName: 🔏 CodeQL 3000 Finalize @@ -153,16 +166,17 @@ jobs: condition: succeededOrFailed() # Explicitly download symbols for the drop since the SDL image doesn't have http://SymWeb access and APIScan cannot handle https yet. - - pwsh: | - $pat = '$(SymbolServerPAT)' - if ($pat -like '*PAT*' -or $pat -eq '') { - throw 'No PAT defined' - } - $url = 'https://microsoft.artifacts.visualstudio.com/defaultcollection/_apis/symbol/symsrv' - $(symbolToolPath) --authenticated-server-path $(SymbolServerPAT) $url --symbols -d "$env:ob_outputDirectory\*" --recurse-subdirectories - displayName: 'Download Symbols for binaries' - retryCountOnTaskFailure: 2 - workingDirectory: '$(repoRoot)' + # - pwsh: | + # $pat = '$(SymbolServerPAT)' + # if ($pat -like '*PAT*' -or $pat -eq '') + # { + # throw 'No PAT defined' + # } + # $url = 'https://microsoft.artifacts.visualstudio.com/defaultcollection/_apis/symbol/symsrv' + # $(symbolToolPath) --authenticated-server-path $(SymbolServerPAT) $url --symbols -d "$env:ob_outputDirectory\*" --recurse-subdirectories + # displayName: 'Download Symbols for binaries' + # retryCountOnTaskFailure: 2 + # workingDirectory: '$(repoRoot)' - pwsh: | Get-ChildItem '$(ob_outputDirectory)' -File -Recurse | @@ -173,16 +187,10 @@ jobs: Md5Hash = (Get-FileHash -Algorithm MD5 -Path $_.FullName).Hash Sha512Hash = (Get-FileHash -Algorithm SHA512 -Path $_.FullName).Hash } - } | Export-Csv -Path '$(Build.SourcesDirectory)/ReleaseFileHash.csv' + } | Export-Csv -Path '$(Build.SourcesDirectory)\ReleaseFileHash.csv' workingDirectory: '$(repoRoot)' displayName: 'Create release file hash artifact' - pwsh: | - Copy-Item -Path '$(Build.SourcesDirectory)/ReleaseFileHash.csv' -Destination '$(ob_outputDirectory)' -Verbose + Copy-Item -Path '$(Build.SourcesDirectory)\ReleaseFileHash.csv' -Destination '$(ob_outputDirectory)' -Verbose displayName: 'Publish Build File Hash artifact' - - - pwsh: | - Get-ChildItem -Path env: | Out-String -width 150 -Stream | write-Verbose -Verbose - displayName: Capture Environment - condition: succeededOrFailed() - workingDirectory: '$(repoRoot)' From 4eac58ee87ddb018adf080f276def08e509725ef Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Fri, 5 Sep 2025 15:49:54 -0700 Subject: [PATCH 5/5] Include all agents in build by default --- tools/metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/metadata.json b/tools/metadata.json index 54f4334f..14857f91 100644 --- a/tools/metadata.json +++ b/tools/metadata.json @@ -1,3 +1,3 @@ { - "AgentsToInclude": ["msaz", "openai-gpt"] + "AgentsToInclude": "*" }