Add support for unstable Python versions (actions#38)

* Add support of unstable versions to package generation (#2)
* Add support of symver versions to Python setup scripts and tests

Co-authored-by: Maksim Petrov <[email protected]>
Co-authored-by: MaksimZhukov <[email protected]>
Co-authored-by: Maxim Lobanov <[email protected]>

Author: 4 people

.github/workflows/manifest-config-validation.yml

@@ -0,0 +1,32 @@
+name: Manifest config tests
+on: 
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - 'versions-manifest.json'
+
+jobs:
+  RunTests:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: true
+
+    - name: Install Pester
+      shell: pwsh
+      run: |
+        Install-Module Pester -Force -Scope CurrentUser
+
+    - name: Run tests
+      shell: pwsh  
+      run: |
+        Import-Module Pester
+        Invoke-Pester -Configuration @{
+          Run = @{
+            Path = "tests/ManifestConfig.Tests.ps1"
+            Exit = $true
+          }
+        }

azure-pipelines/templates/build-job.yml

@@ -14,6 +14,7 @@ jobs:
       targetType: filePath
       filePath: './builders/build-python.ps1'
       arguments: '-Version $(VERSION) -Platform $(Platform) -Architecture $(Architecture)'
+      pwsh: true
 
   - task: PublishPipelineArtifact@1
     displayName: 'Publish Artifact: Python $(VERSION)'

azure-pipelines/templates/test-job.yml

@@ -3,6 +3,8 @@ jobs:
   pool: 
     name: Azure Pipelines
     vmImage: $(VmImage)
+  variables:
+    TestRunTitle: 'python-$(Platform)-$(Architecture)'
   steps:
   - checkout: self
     submodules: true

builders/build-python.ps1

@@ -21,14 +21,15 @@ Required parameter. The platform for which Python will be built.
 #>
 
 param(
-    [Parameter (Mandatory=$true)][Version] $Version,
+    [Parameter (Mandatory=$true)][semver] $Version,
     [Parameter (Mandatory=$true)][string] $Platform,
     [string] $Architecture = "x64"
 )
 
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "common-helpers.psm1") -DisableNameChecking
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "nix-helpers.psm1") -DisableNameChecking
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "win-helpers.psm1") -DisableNameChecking
+Import-Module (Join-Path $PSScriptRoot "python-version.psm1") -DisableNameChecking
 
 function Get-PythonBuilder {
     <#
@@ -49,13 +50,12 @@ function Get-PythonBuilder {
 
     #>
 
-    param (
-        [version] $Version,
+    param(
+        [semver] $Version,
         [string] $Architecture,
         [string] $Platform
     )
 
-    $Platform = $Platform.ToLower()  
     if ($Platform -match 'win32') {
         $builder = [WinPythonBuilder]::New($Version, $Architecture, $Platform)
     } elseif ($Platform -match 'linux') {
@@ -71,5 +71,5 @@ function Get-PythonBuilder {
 }
 
 ### Create Python builder instance, and build artifact
-$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture  -Platform $Platform 
+$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture -Platform $Platform 
 $Builder.Build()

builders/macos-python-builder.psm1

@@ -17,7 +17,7 @@ class macOSPythonBuilder : NixPythonBuilder {
     #>
 
     macOSPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) { }

builders/nix-python-builder.psm1

@@ -33,13 +33,12 @@ class NixPythonBuilder : PythonBuilder {
     [string] $OutputArtifactName
 
     NixPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) {
         $this.InstallationTemplateName = "nix-setup-template.sh"	
         $this.InstallationScriptName = "setup.sh"
-
         $this.OutputArtifactName = "python-$Version-$Platform-$Architecture.tar.gz"
     }
 
@@ -50,8 +49,10 @@ class NixPythonBuilder : PythonBuilder {
         #>
 
         $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $nativeVersion = Convert-Version -version $this.Version
 
-        return "${base}/$($this.Version)/Python-$($this.Version).tgz"
+        return "${base}/${versionName}/Python-${nativeVersion}.tgz"
     }
 
     [string] GetPythonBinary() {
@@ -95,9 +96,7 @@ class NixPythonBuilder : PythonBuilder {
         $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
 
         $variablesToReplace = @{
-            "{{__VERSION_MAJOR__}}" = $this.Version.Major;
-            "{{__VERSION_MINOR__}}" = $this.Version.Minor;
-            "{{__VERSION_BUILD__}}" = $this.Version.Build;
+            "{{__VERSION_FULL__}}" = $this.Version;
         }
         $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
 

builders/python-builder.psm1

@@ -29,7 +29,7 @@ class PythonBuilder {
 
     #>
 
-    [version] $Version
+    [semver] $Version
     [string] $Architecture
     [string] $Platform
     [string] $HostedToolcacheLocation
@@ -38,17 +38,17 @@ class PythonBuilder {
     [string] $ArtifactFolderLocation
     [string] $InstallationTemplatesLocation
 
-    PythonBuilder ([version] $version, [string] $architecture, [string] $platform) {
-        $this.Version = $version
-        $this.Architecture = $architecture
-        $this.Platform = $platform
+    PythonBuilder ([semver] $version, [string] $architecture, [string] $platform) {
+        $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers"
 
         $this.HostedToolcacheLocation = $env:AGENT_TOOLSDIRECTORY
         $this.TempFolderLocation = $env:BUILD_SOURCESDIRECTORY
         $this.WorkFolderLocation = $env:BUILD_BINARIESDIRECTORY
         $this.ArtifactFolderLocation = $env:BUILD_STAGINGDIRECTORY
 
-        $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers"
+        $this.Version = $version
+        $this.Architecture = $architecture
+        $this.Platform = $platform
     }
 
     [uri] GetBaseUri() {
@@ -79,11 +79,21 @@ class PythonBuilder {
         return "$pythonToolcacheLocation/$($this.Version)/$($this.Architecture)"
     }
 
+    [string] GetBaseVersion() {
+        <#
+        .SYNOPSIS
+        Return Major.Minor.Patch version string.
+        #>
+
+        return "$($this.Version.Major).$($this.Version.Minor).$($this.Version.Patch)"
+    }
+
     [void] PreparePythonToolcacheLocation() {
         <#
         .SYNOPSIS
         Prepare system hostedtoolcache folder for new Python version. 
         #>
+        
         $pythonBinariesLocation = $this.GetFullPythonToolcacheLocation()
 
         if (Test-Path $pythonBinariesLocation) {

builders/python-version.psm1

@@ -0,0 +1,45 @@
+function Convert-Label() {
+    <#
+    .SYNOPSIS
+    Convert generic semver label to native Python label.
+    #>
+
+    param(
+        [Parameter(Mandatory)]
+        [string] $label
+    )
+
+    switch ($label) {
+        "alpha" { return "a" }
+        "beta" { return "b" }
+        "rc" { return "rc" }
+        default { throw "Invalid version label '$label'" }
+    }
+}
+
+function Convert-Version {
+    <#
+    .SYNOPSIS
+    Convert generic semver version to native Python version.
+    #>
+
+    param(
+        [Parameter(Mandatory)]
+        [semver] $version,
+        [char] $delimiter = "."
+    )
+
+    $nativeVersion = "{0}.{1}.{2}" -f $version.Major, $version.Minor, $version.Patch
+
+    if ($version.PreReleaseLabel)
+    {
+        $preReleaseLabel = $version.PreReleaseLabel.Split($delimiter)
+        
+        $preReleaseLabelName = Convert-Label -Label $preReleaseLabel[0]
+        $preReleaseLabelVersion = $preReleaseLabel[1]
+
+        $nativeVersion += "${preReleaseLabelName}${preReleaseLabelVersion}"
+    }
+
+    return $nativeVersion
+}

builders/ubuntu-python-builder.psm1

@@ -17,7 +17,7 @@ class UbuntuPythonBuilder : NixPythonBuilder {
     #>
 
     UbuntuPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) { }
@@ -68,6 +68,7 @@ class UbuntuPythonBuilder : NixPythonBuilder {
         } else {
             $tkinterInstallString = "sudo apt install -y python-tk tk-dev"
         }
+
         Execute-Command -Command $tkinterInstallString
 
         ### Install dependent packages

builders/win-python-builder.psm1

@@ -27,7 +27,7 @@ class WinPythonBuilder : PythonBuilder {
     [string] $OutputArtifactName
 
     WinPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) {
@@ -72,10 +72,12 @@ class WinPythonBuilder : PythonBuilder {
         #>
 
         $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $nativeVersion = Convert-Version -version $this.Version
         $architecture = $this.GetArchitectureExtension()
         $extension = $this.GetPythonExtension()
 
-        $uri = "${base}/$($this.Version)/python-$($this.Version)${architecture}${extension}"
+        $uri = "${base}/${versionName}/python-${nativeVersion}${architecture}${extension}"
 
         return $uri
     }