Skip to content

Commit 05fe9de

Browse files
authored
Add SiteExtension.proj to replace build-extension.ps1 (#10168)
* Add SiteExtension.proj * Publish workers, hash hard links * Add workers.props, avoid publishing worker files * Refactor how worker is packaged * Add PrivateSiteExtension * Fix duplicate build and file write error * Comment and split out powershell regex match * Fix MoveSymbols * Publish PrivateSiteExtension * Remove Python worker from private extension * Zip site extension artifacts * Clean dotnet worker files * Default ZipArtifactsPath to PackageOutputPath * Add comments, update dotnet sdk * Add explicit CI restore stage * Split out build stage as well * Add python.props back in with redirection message * Fix integration build issue * Fix symbols output * Revert python.props changes * Revert global.json change * Extract build site extension into template * Remove build-extension.ps1 * Add build-extensions.ps1 back, with error directing to replacement * Fix log publishing * Fix check-vuln placement * Fix deletion of code-sign summary files * Fix symbol SBOM and artifact * Add single/multi-tfm support for SiteExtension.proj * Fix end-of-file line breaks * Extract Workers to their own props files * Fix ZipPublish when zip not specified * Remove redundant python removal * .proj -> .csproj * Remove minorVersionPrefix * Update CODEOWNERS * Add EnsureWorkersFolder target * Fix windows build * Pack linux artifact * Add release config * Fix nuget signing path * Add condition for hard link hashes * Add IsZippable check and readme
1 parent cc01cfa commit 05fe9de

28 files changed

+596
-511
lines changed

CODEOWNERS

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,9 @@
99

1010
# Adding codeowner for Python specific files such that GitHub automatically adds python folks as a reviewer.
1111
build/python.props @vrdmr @gavin-aguiar @YunchuWang @pdthummar @hallvictoria
12+
eng/build/Workers.Python.props @vrdmr @gavin-aguiar @YunchuWang @pdthummar @hallvictoria
1213

1314
# Deps.json validation file
1415
test/WebJobs.Script.Tests/Microsoft.Azure.WebJobs.Script.WebHost.deps.json @fabiocav @brettsam @mathewc
1516

16-
src/WebJobs.Script.WebHost/PreJIT/* @vrdmr @pragnagopa @eliaslopezgt @VpOfEngineering @azure/azure-functions-core
17-
18-
# CI owners
19-
20-
eng/ci/* @fabiocav @jviau @brettsam
17+
src/WebJobs.Script.WebHost/PreJIT/* @vrdmr @pragnagopa @eliaslopezgt @azure/azure-functions-core

Directory.Build.props

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
<PropertyGroup>
1212
<RepoRoot>$(MSBuildThisFileDirectory)</RepoRoot>
1313
<EngRoot>$(RepoRoot)eng/</EngRoot>
14+
<TargetsRoot>$(EngRoot)build/</TargetsRoot>
1415
</PropertyGroup>
1516

1617
<PropertyGroup>

Directory.Build.targets

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
<Project>
22

3-
<Import Project="$(EngRoot)build/Engineering.targets" />
3+
<Import Project="$(TargetsRoot)Engineering.targets" />
44

55
</Project>

build/build-extensions.ps1

Lines changed: 1 addition & 219 deletions
Original file line numberDiff line numberDiff line change
@@ -1,219 +1 @@
1-
param (
2-
[string]$buildNumber = "0",
3-
[string]$suffix = "",
4-
[ValidateSet("6", "8", "")][string]$minorVersionPrefix = "",
5-
[string]$hashesForHardlinksFile = "hashesForHardlinks.txt"
6-
)
7-
8-
Import-Module "$PSScriptRoot\Get-AzureFunctionsVersion.psm1" -Force
9-
$rootDir = Split-Path -Parent $PSScriptRoot
10-
$outDir = "$rootDir\out"
11-
$publishDir = "$outDir\pub\WebJobs.Script.WebHost"
12-
13-
$extensionVersion = Get-AzureFunctionsVersion $buildNumber $suffix $minorVersionPrefix
14-
Write-Host "Site extension version: $extensionVersion"
15-
16-
function ZipContent([string] $sourceDirectory, [string] $target) {
17-
$stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
18-
19-
Write-Host "======================================"
20-
Write-Host "Zipping $sourceDirectory into $target"
21-
22-
if (Test-Path $target) {
23-
Remove-Item $target
24-
}
25-
26-
Add-Type -assembly "system.io.compression.filesystem"
27-
[IO.Compression.ZipFile]::CreateFromDirectory($sourceDirectory, $target)
28-
29-
Write-Host "Done zipping $target. Elapsed: $($stopwatch.Elapsed)"
30-
Write-Host "======================================"
31-
Write-Host ""
32-
}
33-
34-
function BuildRuntime([string] $targetRid, [bool] $isSelfContained) {
35-
$stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
36-
37-
$publishTarget = "$publishDir\release_$targetRid"
38-
$projectPath = "$rootDir\src\WebJobs.Script.WebHost\WebJobs.Script.WebHost.csproj"
39-
if (-not (Test-Path $projectPath))
40-
{
41-
throw "Project path '$projectPath' does not exist."
42-
}
43-
44-
$cmd = "publish", $projectPath , "-r", "$targetRid", "--self-contained", "$isSelfContained", "-v", "m", "-c", "Release", "-p:IsPackable=false", "-p:BuildNumber=$buildNumber", "-p:MinorVersionPrefix=$minorVersionPrefix"
45-
46-
Write-Host "======================================"
47-
Write-Host "Building $targetRid"
48-
Write-Host " Self-Contained: $isSelfContained"
49-
Write-Host " Publish Directory: $publishTarget"
50-
Write-Host ""
51-
Write-Host "dotnet $cmd"
52-
Write-Host ""
53-
54-
& dotnet $cmd
55-
56-
if ($LASTEXITCODE -ne 0)
57-
{
58-
exit $LASTEXITCODE
59-
}
60-
61-
Write-Host ""
62-
$symbols = Get-ChildItem -Path $publishTarget -Filter *.pdb
63-
$symbols += Get-ChildItem -Path "$publishTarget\workers\dotnet-isolated\*" -Include "*.pdb", "*.dbg" -Recurse
64-
Write-Host "Zipping symbols: $($symbols.Count) symbols found"
65-
66-
$symbolsPath = "$publishDir\Symbols"
67-
if (!(Test-Path -PathType Container $symbolsPath)) {
68-
New-Item -ItemType Directory -Path $symbolsPath | Out-Null
69-
}
70-
71-
$symbols | Compress-Archive -DestinationPath "$symbolsPath\Functions.Symbols.$extensionVersion.$targetRid.zip" | Out-Null
72-
$symbols | Remove-Item | Out-Null
73-
74-
Write-Host ""
75-
CleanOutput $publishTarget
76-
Write-Host ""
77-
Write-Host "Done building $targetRid. Elapsed: $($stopwatch.Elapsed)"
78-
Write-Host "======================================"
79-
Write-Host ""
80-
}
81-
82-
function GetFolderSizeInMb([string] $rootPath) {
83-
return [math]::Round((Get-ChildItem $rootPath -Recurse | Measure-Object -Property Length -Sum -ErrorAction Stop).Sum / 1Mb, 2)
84-
}
85-
86-
function CleanOutput([string] $rootPath) {
87-
Write-Host "Cleaning build output under $rootPath"
88-
Write-Host " Current size: $(GetFolderSizeInMb $rootPath) Mb"
89-
90-
Write-Host " Removing any linux and osx runtimes"
91-
Remove-Item -Recurse -Force "$privateSiteExtensionPath\$bitness\runtimes\linux" -ErrorAction SilentlyContinue
92-
Remove-Item -Recurse -Force "$privateSiteExtensionPath\$bitness\runtimes\osx" -ErrorAction SilentlyContinue
93-
94-
Write-Host " Removing python worker"
95-
Remove-Item -Recurse -Force "$rootPath\workers\python" -ErrorAction SilentlyContinue
96-
97-
$keepRuntimes = @('win', 'win-x86', 'win10-x86', 'win-x64', 'win10-x64')
98-
Write-Host " Removing all powershell runtimes except $keepRuntimes"
99-
Get-ChildItem "$rootPath\workers\powershell" -Directory -ErrorAction SilentlyContinue |
100-
ForEach-Object { Get-ChildItem "$($_.FullName)\runtimes" -Directory -Exclude $keepRuntimes } |
101-
Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
102-
103-
Write-Host " Removing FunctionsNetHost(linux executable) and dependencies from dotnet-isolated worker"
104-
$dotnetIsolatedBinPath = Join-Path $rootPath "workers\dotnet-isolated\bin"
105-
if (Test-Path $dotnetIsolatedBinPath) {
106-
Remove-Item -Path (Join-Path $dotnetIsolatedBinPath "FunctionsNetHost") -ErrorAction SilentlyContinue
107-
Get-ChildItem -Path $dotnetIsolatedBinPath -Filter "*.so" | Remove-Item -ErrorAction SilentlyContinue
108-
}
109-
110-
Write-Host " Current size: $(GetFolderSizeInMb $rootPath) Mb"
111-
}
112-
113-
function CreateSiteExtensions() {
114-
$stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
115-
$siteExtensionPath = "$publishDir\temp_extension"
116-
117-
if (Test-Path $siteExtensionPath) {
118-
Write-Host " Existing site extension path found. Deleting."
119-
Remove-Item $siteExtensionPath -Recurse -Force | Out-Null
120-
}
121-
122-
# The official site extension needs to be nested inside a folder with its version.
123-
# Not using the suffix (eg: '-ci') here as it may not work correctly in a private stamp
124-
$officialSiteExtensionPath = "$siteExtensionPath\$extensionVersion"
125-
126-
Write-Host "======================================"
127-
Write-Host "Copying build to temp directory to prepare for zipping official site extension."
128-
Copy-Item -Path $publishDir\release_win-x86\ -Destination $officialSiteExtensionPath\32bit -Force -Recurse > $null
129-
Copy-Item -Path $publishDir\release_win-x64 -Destination $officialSiteExtensionPath\64bit -Force -Recurse > $null
130-
Copy-Item -Path $officialSiteExtensionPath\32bit\applicationHost.xdt -Destination $officialSiteExtensionPath -Force > $null
131-
Write-Host " Deleting workers directory: $officialSiteExtensionPath\32bit\workers"
132-
Remove-Item -Recurse -Force "$officialSiteExtensionPath\32bit\workers" -ErrorAction SilentlyContinue
133-
Write-Host " Moving workers directory: $officialSiteExtensionPath\64bit\workers to $officialSiteExtensionPath\workers"
134-
Move-Item -Path "$officialSiteExtensionPath\64bit\workers" -Destination "$officialSiteExtensionPath\workers"
135-
136-
# This goes in the root dir
137-
Copy-Item $rootDir\src\WebJobs.Script.WebHost\extension.xml $siteExtensionPath > $null
138-
139-
Write-Host "Done copying. Elapsed: $($stopwatch.Elapsed)"
140-
Write-Host "======================================"
141-
Write-Host ""
142-
143-
Write-Host "======================================"
144-
Write-Host "Generating hashes for hard links"
145-
WriteHashesFile $siteExtensionPath/$extensionVersion
146-
Write-Host "Done generating hashes for hard links into $siteExtensionPath/$extensionVersion"
147-
Write-Host "======================================"
148-
Write-Host
149-
150-
$zipOutput = "$publishDir\SiteExtension"
151-
$hashesForHardLinksPath = "$siteExtensionPath\$extensionVersion\$hashesForHardlinksFile"
152-
New-Item -Itemtype directory -path $zipOutput -Force > $null
153-
if ($minorVersionPrefix -eq "") {
154-
ZipContent $siteExtensionPath "$zipOutput\Functions.$extensionVersion.zip"
155-
} elseif ($minorVersionPrefix -eq "8") {
156-
Write-Host "======================================"
157-
# Only the "Functions" site extension supports hard links
158-
Write-Host "MinorVersionPrefix is '8'. Removing $hashesForHardLinksPath before zipping."
159-
Remove-Item -Force "$hashesForHardLinksPath" -ErrorAction Stop
160-
# The .NET 8 host doesn't require any workers. Doing this to save space.
161-
Write-Host "Removing workers before zipping."
162-
# The host requires that this folder exists and it cannot be empty
163-
Remove-Item -Recurse -Force "$siteExtensionPath\$extensionVersion\workers" -ErrorAction Stop
164-
New-Item -Path "$siteExtensionPath\$extensionVersion" -Name "workers" -ItemType Directory -ErrorAction Stop | Out-Null
165-
Set-Content -Force -Path "$siteExtensionPath\$extensionVersion\workers\this_folder_intentionally_empty.txt" -Value ".NET 8 builds do not have workers. However, this folder must contain at least one file." -ErrorAction Stop
166-
Write-Host "======================================"
167-
Write-Host
168-
ZipContent $siteExtensionPath "$zipOutput\FunctionsInProc8.$extensionVersion.zip"
169-
} elseif ($minorVersionPrefix -eq "6") {
170-
# Only the "Functions" site extension supports hard links
171-
Write-Host "======================================"
172-
Write-Host "MinorVersionPrefix is '6'. Removing $hashesForHardLinksPath before zipping."
173-
Remove-Item -Force "$hashesForHardLinksPath" -ErrorAction Stop
174-
Write-Host "======================================"
175-
Write-Host
176-
ZipContent $siteExtensionPath "$zipOutput\FunctionsInProc.$extensionVersion.zip"
177-
}
178-
179-
Remove-Item $siteExtensionPath -Recurse -Force > $null
180-
181-
Write-Host "======================================"
182-
$stopwatch.Reset()
183-
Write-Host "Copying build to temp directory to prepare for zipping private site extension."
184-
Copy-Item -Path $publishDir\release_win-x86\ -Destination $siteExtensionPath\SiteExtensions\Functions\32bit -Force -Recurse > $null
185-
Copy-Item -Path $siteExtensionPath\SiteExtensions\Functions\32bit\applicationHost.xdt -Destination $siteExtensionPath\SiteExtensions\Functions -Force > $null
186-
Write-Host "Done copying. Elapsed: $($stopwatch.Elapsed)"
187-
Write-Host "======================================"
188-
Write-Host ""
189-
190-
$zipOutput = "$publishDir\PrivateSiteExtension"
191-
New-Item -Itemtype directory -path $zipOutput -Force > $null
192-
ZipContent $siteExtensionPath "$zipOutput\Functions.Private.$extensionVersion.win-x32.inproc.zip"
193-
194-
Remove-Item $siteExtensionPath -Recurse -Force > $null
195-
}
196-
197-
function WriteHashesFile([string] $directoryPath) {
198-
New-Item -Path "$directoryPath/../temp_hashes" -ItemType Directory | Out-Null
199-
$temp_current = (Get-Location)
200-
Set-Location $directoryPath
201-
Get-ChildItem -Recurse $directoryPath | Where-Object { $_.PsIsContainer -eq $false } | Foreach-Object { "Hash:" + [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((Get-FileHash -Algorithm MD5 $_.FullName).Hash)) + " FileName:" + (Resolve-Path -Relative -Path $_.FullName) } | Out-File -FilePath "$directoryPath\..\temp_hashes\$hashesForHardlinksFile"
202-
Move-Item -Path "$directoryPath/../temp_hashes/$hashesForHardlinksFile" -Destination "$directoryPath" -Force
203-
Set-Location $temp_current
204-
Remove-Item "$directoryPath/../temp_hashes" -Recurse -Force > $null
205-
}
206-
207-
Write-Host "Output directory: $publishDir"
208-
if (Test-Path $publishDir) {
209-
Write-Host " Existing build output found. Deleting."
210-
Remove-Item $publishDir -Recurse -Force -ErrorAction Stop
211-
}
212-
213-
Write-Host "Extensions version: $extensionVersion"
214-
Write-Host ""
215-
216-
BuildRuntime "win-x86"
217-
BuildRuntime "win-x64"
218-
219-
CreateSiteExtensions
1+
Write-Error "This script is no longer used. Instead, publish src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj directly."

build/common.props

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,12 @@
88
<PatchVersion>0</PatchVersion>
99
<BuildNumber Condition="'$(BuildNumber)' == '' ">0</BuildNumber>
1010
<PreviewVersion></PreviewVersion>
11-
11+
1212
<!-- During previews, always generate this suffix, even for official releases -->
1313
<_VersionSuffix Condition="'$(PreviewVersion)' != ''">-preview.$(PreviewVersion).$(BuildNumber)</_VersionSuffix>
1414
<!-- Otherwise, use the suffix directly, adding a '-' -->
1515
<_VersionSuffix Condition="'$(PreviewVersion)' == '' and '$(VersionSuffix)' != ''">-$(VersionSuffix)</_VersionSuffix>
16-
16+
1717
<VersionPrefix>$(MajorVersion).$(MinorVersion).$(PatchVersion)</VersionPrefix>
1818
<Version>$(VersionPrefix)$(_VersionSuffix)</Version>
1919
<AssemblyVersion>$(MajorVersion).$(MinorVersion).0.0</AssemblyVersion>

build/python.props

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
<Project>
2-
<ItemGroup>
3-
<PackageReference Include="Microsoft.Azure.Functions.PythonWorker" Version="4.34.0" />
4-
</ItemGroup>
2+
<!-- See /eng/build/Workers.Python.props -->
3+
<Target Name="_DoNotImport" BeforeTargets="Build">
4+
<Error Text="This file should not be imported. Import '$(TargetsRoot)Workers.props'." />
5+
</Target>
56
</Project>

0 commit comments

Comments
 (0)