Merge branch 'master' of github.com:python/cpython into fix-issue21914

* 'master' of github.com:python/cpython: (2949 commits)
  Add files in tests/test_peg_generator to the install target lists (GH-19723)
  bpo-40398: Fix typing.get_args() for special generic aliases. (GH-19720)
  bpo-40348: Fix typos in the programming FAQ (GH-19729)
  bpo-38387: Formally document PyDoc_STRVAR and PyDoc_STR macros (GH-16607)
  bpo-40401: Remove duplicate pyhash.h include from pythoncore.vcxproj (GH-19725)
  bpo-40387: Improve queue join() example. (GH-19724)
  bpo-40396: Support GenericAlias in the typing functions. (GH-19718)
  Fix typo in Lib/typing.py (GH-19717)
  Fix typo in object.__format__ docs (GH-19504)
  bpo-40275: Avoid importing logging in test.support (GH-19601)
  bpo-40275: Avoid importing socket in test.support (GH-19603)
  bpo-40275: Avoid importing asyncio in test.support (GH-19600)
  bpo-40279: Add some error-handling to the module initialisation docs example (GH-19705)
  closes bpo-40385: Remove Tools/scripts/checkpyc.py (GH-19709)
  bpo-40334: Add What's New sections for PEP 617 and PEP 585 (GH-19704)
  bpo-40340: Separate examples more clearly in the programming FAQ (GH-19688)
  bpo-40360: Deprecate lib2to3 module in light of PEP 617 (GH-19663)
  bpo-40334: Rewrite test_c_parser to avoid memory leaks (GH-19694)
  bpo-38061: subprocess uses closefrom() on FreeBSD (GH-19697)
  bpo-38061: os.closerange() uses closefrom() on FreeBSD (GH-19696)
  ...

Author: CuriousLearner

.azure-pipelines/ci.yml

@@ -35,14 +35,16 @@ jobs:
 - job: macOS_CI_Tests
   displayName: macOS CI Tests
   dependsOn: Prebuild
-  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+  #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+  # bpo-39837: macOS tests on Azure Pipelines are disabled
+  condition: false
 
   variables:
     testRunTitle: '$(build.sourceBranchName)-macos'
     testRunPlatform: macos
 
   pool:
-    vmImage: xcode9-macos10.13
+    vmImage: macos-10.14
 
   steps:
   - template: ./macos-steps.yml
@@ -59,7 +61,7 @@ jobs:
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.0j
+    openssl_version: 1.1.1f
 
   steps:
   - template: ./posix-steps.yml
@@ -116,7 +118,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.0j
+    openssl_version: 1.1.1f
 
   steps:
   - template: ./posix-steps.yml
@@ -131,7 +133,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: vs2017-win2016
+    vmImage: windows-2019
 
   strategy:
     matrix:
@@ -145,7 +147,7 @@ jobs:
         buildOpt: '-p x64'
         testRunTitle: '$(Build.SourceBranchName)-win64'
         testRunPlatform: win64
-    maxParallel: 2
+    maxParallel: 4
 
   steps:
   - template: ./windows-steps.yml

.azure-pipelines/docs-steps.yml

@@ -12,7 +12,7 @@ steps:
   inputs:
     versionSpec: '>=3.6'
 
-- script: python -m pip install sphinx==1.8.2 blurb python-docs-theme
+- script: python -m pip install sphinx==2.2.0 blurb python-docs-theme
   displayName: 'Install build dependencies'
 
 - ${{ if ne(parameters.latex, 'true') }}:
@@ -21,7 +21,7 @@ steps:
     displayName: 'Build documentation'
 
 - ${{ if eq(parameters.latex, 'true') }}:
-  - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full 
+  - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full
     displayName: 'Install LaTeX'
 
   - script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb'

.azure-pipelines/macos-steps.yml

@@ -6,14 +6,16 @@ steps:
 - script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-azdev
   displayName: 'Configure CPython (debug)'
 
-- script: make -s -j4
+- script: make -j4
   displayName: 'Build CPython'
 
 - script: make pythoninfo
   displayName: 'Display build info'
 
 - script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
   displayName: 'Tests'
+  continueOnError: true
+  timeoutInMinutes: 30
 
 - task: PublishTestResults@2
   displayName: 'Publish Test Results'

.azure-pipelines/posix-deps-apt.sh

@@ -1,3 +1,4 @@
+#!/bin/sh
 apt-get update
 
 apt-get -yq install \

.azure-pipelines/posix-steps.yml

@@ -20,7 +20,7 @@ steps:
 - script: ./configure --with-pydebug
   displayName: 'Configure CPython (debug)'
 
-- script: make -s -j4
+- script: make -j4
   displayName: 'Build CPython'
 
 - ${{ if eq(parameters.coverage, 'true') }}:
@@ -49,7 +49,7 @@ steps:
   - script: ./venv/bin/python -m coverage xml
     displayName: 'Generate coverage.xml'
 
-  - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash)
+  - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml
     displayName: 'Publish code coverage results'
 
 

.azure-pipelines/pr.yml

@@ -33,14 +33,16 @@ jobs:
 - job: macOS_PR_Tests
   displayName: macOS PR Tests
   dependsOn: Prebuild
-  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+  #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+  # bpo-39837: macOS tests on Azure Pipelines are disabled
+  condition: false
 
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-macos'
     testRunPlatform: macos
 
   pool:
-    vmImage: xcode9-macos10.13
+    vmImage: macos-10.14
 
   steps:
   - template: ./macos-steps.yml
@@ -59,7 +61,7 @@ jobs:
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.0j
+    openssl_version: 1.1.1f
 
   steps:
   - template: ./posix-steps.yml
@@ -116,7 +118,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.0j
+    openssl_version: 1.1.1f
 
   steps:
   - template: ./posix-steps.yml
@@ -131,7 +133,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: vs2017-win2016
+    vmImage: windows-2019
 
   strategy:
     matrix:
@@ -145,7 +147,10 @@ jobs:
         buildOpt: '-p x64'
         testRunTitle: '$(System.PullRequest.TargetBranch)-win64'
         testRunPlatform: win64
-    maxParallel: 2
+      winarm64:
+        arch: arm64
+        buildOpt: '-p arm64'
+    maxParallel: 4
 
   steps:
   - template: ./windows-steps.yml

.azure-pipelines/windows-release.yml

@@ -0,0 +1,129 @@
+name: Release_$(Build.SourceBranchName)_$(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+variables:
+    __RealSigningCertificate: 'Python Software Foundation'
+# QUEUE TIME VARIABLES
+#   GitRemote: python
+#   SourceTag:
+#   DoPGO: true
+#   SigningCertificate: 'Python Software Foundation'
+#   SigningDescription: 'Built: $(Build.BuildNumber)'
+#   DoLayout: true
+#   DoMSIX: true
+#   DoNuget: true
+#   DoEmbed: true
+#   DoMSI: true
+#   DoPublish: false
+#   PyDotOrgUsername: ''
+#   PyDotOrgServer: ''
+#   BuildToPublish: ''
+
+trigger: none
+pr: none
+
+stages:
+- stage: Build
+  displayName: Build binaries
+  condition: and(succeeded(), not(variables['BuildToPublish']))
+  jobs:
+  - template: windows-release/stage-build.yml
+
+- stage: Sign
+  displayName: Sign binaries
+  dependsOn: Build
+  condition: and(succeeded(), not(variables['BuildToPublish']))
+  jobs:
+  - template: windows-release/stage-sign.yml
+
+- stage: Layout
+  displayName: Generate layouts
+  dependsOn: Sign
+  condition: and(succeeded(), not(variables['BuildToPublish']))
+  jobs:
+  - template: windows-release/stage-layout-full.yml
+  - template: windows-release/stage-layout-embed.yml
+  - template: windows-release/stage-layout-nuget.yml
+
+- stage: Pack
+  dependsOn: Layout
+  condition: and(succeeded(), not(variables['BuildToPublish']))
+  jobs:
+  - template: windows-release/stage-pack-nuget.yml
+
+- stage: Test
+  dependsOn: Pack
+  condition: and(succeeded(), not(variables['BuildToPublish']))
+  jobs:
+  - template: windows-release/stage-test-embed.yml
+  - template: windows-release/stage-test-nuget.yml
+
+- stage: Layout_MSIX
+  displayName: Generate MSIX layouts
+  dependsOn: Sign
+  condition: and(succeeded(), and(eq(variables['DoMSIX'], 'true'), not(variables['BuildToPublish'])))
+  jobs:
+  - template: windows-release/stage-layout-msix.yml
+
+- stage: Pack_MSIX
+  displayName: Package MSIX
+  dependsOn: Layout_MSIX
+  condition: and(succeeded(), not(variables['BuildToPublish']))
+  jobs:
+  - template: windows-release/stage-pack-msix.yml
+
+- stage: Build_MSI
+  displayName: Build MSI installer
+  dependsOn: Sign
+  condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), not(variables['BuildToPublish'])))
+  jobs:
+  - template: windows-release/stage-msi.yml
+
+- stage: Test_MSI
+  displayName: Test MSI installer
+  dependsOn: Build_MSI
+  condition: and(succeeded(), not(variables['BuildToPublish']))
+  jobs:
+  - template: windows-release/stage-test-msi.yml
+
+- stage: PublishPyDotOrg
+  displayName: Publish to python.org
+  dependsOn: ['Test_MSI', 'Test']
+  condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish'])))
+  jobs:
+  - template: windows-release/stage-publish-pythonorg.yml
+
+- stage: PublishNuget
+  displayName: Publish to nuget.org
+  dependsOn: Test
+  condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish'])))
+  jobs:
+  - template: windows-release/stage-publish-nugetorg.yml
+
+- stage: PublishStore
+  displayName: Publish to Store
+  dependsOn: Pack_MSIX
+  condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish'])))
+  jobs:
+  - template: windows-release/stage-publish-store.yml
+
+
+- stage: PublishExistingPyDotOrg
+  displayName: Publish existing build to python.org
+  dependsOn: []
+  condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish']))
+  jobs:
+  - template: windows-release/stage-publish-pythonorg.yml
+
+- stage: PublishExistingNuget
+  displayName: Publish existing build to nuget.org
+  dependsOn: []
+  condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish']))
+  jobs:
+  - template: windows-release/stage-publish-nugetorg.yml
+
+- stage: PublishExistingStore
+  displayName: Publish existing build to Store
+  dependsOn: []
+  condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish']))
+  jobs:
+  - template: windows-release/stage-publish-store.yml

.azure-pipelines/windows-release/build-steps.yml

@@ -0,0 +1,84 @@
+parameters:
+  ShouldPGO: false
+
+steps:
+- template: ./checkout.yml
+
+- powershell: |
+    $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }};
+    Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)"
+    Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)"
+    Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)"
+    Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)"
+    Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)"
+    Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)-$(Name)"
+  displayName: 'Extract version numbers'
+
+- ${{ if eq(parameters.ShouldPGO, 'false') }}:
+  - powershell: |
+      $env:SigningCertificate = $null
+      .\PCbuild\build.bat -v -p $(Platform) -c $(Configuration)
+    displayName: 'Run build'
+    env:
+      IncludeUwp: true
+      Py_OutDir: '$(Build.BinariesDirectory)\bin'
+
+- ${{ if eq(parameters.ShouldPGO, 'true') }}:
+  - powershell: |
+      $env:SigningCertificate = $null
+      .\PCbuild\build.bat -v -p $(Platform) --pgo
+    displayName: 'Run build with PGO'
+    env:
+      IncludeUwp: true
+      Py_OutDir: '$(Build.BinariesDirectory)\bin'
+
+- powershell: |
+    $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10
+    $tool = (gci -r "$kitroot\Bin\*\x64\signtool.exe" | sort FullName -Desc | select -First 1)
+    if (-not $tool) {
+      throw "SDK is not available"
+    }
+    Write-Host "##vso[task.prependpath]$($tool.Directory)"
+  displayName: 'Add WinSDK tools to path'
+
+- powershell: |
+    $env:SigningCertificate = $null
+    $(_HostPython) PC\layout -vv -b "$(Build.BinariesDirectory)\bin" -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default --arch $(Arch)
+    makecat "${env:CAT}.cdf"
+    del "${env:CAT}.cdf"
+    if (-not (Test-Path "${env:CAT}.cat")) {
+      throw "Failed to build catalog file"
+    }
+  displayName: 'Generate catalog'
+  env:
+    CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python
+    PYTHON_HEXVERSION: $(VersionHex)
+
+- task: PublishPipelineArtifact@0
+  displayName: 'Publish binaries'
+  condition: and(succeeded(), not(and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate'])))
+  inputs:
+    targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)'
+    artifactName: bin_$(Name)
+
+- task: PublishPipelineArtifact@0
+  displayName: 'Publish binaries for signing'
+  condition: and(succeeded(), and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate']))
+  inputs:
+    targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)'
+    artifactName: unsigned_bin_$(Name)
+
+- task: CopyFiles@2
+  displayName: 'Layout Artifact: symbols'
+  inputs:
+    sourceFolder: $(Build.BinariesDirectory)\bin\$(Arch)
+    targetFolder: $(Build.ArtifactStagingDirectory)\symbols\$(Name)
+    flatten: true
+    contents: |
+      **\*.pdb
+
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish Artifact: symbols'
+  inputs:
+    PathToPublish: '$(Build.ArtifactStagingDirectory)\symbols'
+    ArtifactName: symbols