diff --git a/.azure-pipelines/windows-release.yml b/.azure-pipelines/windows-release.yml new file mode 100644 index 00000000000000..3d072e3b43e17e --- /dev/null +++ b/.azure-pipelines/windows-release.yml @@ -0,0 +1,129 @@ +name: Release_$(Build.SourceBranchName)_$(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr) + +variables: + __RealSigningCertificate: 'Python Software Foundation' +# QUEUE TIME VARIABLES +# GitRemote: python +# SourceTag: +# DoPGO: true +# SigningCertificate: 'Python Software Foundation' +# SigningDescription: 'Built: $(Build.BuildNumber)' +# DoLayout: true +# DoMSIX: true +# DoNuget: true +# DoEmbed: true +# DoMSI: true +# DoPublish: false +# PyDotOrgUsername: '' +# PyDotOrgServer: '' +# BuildToPublish: '' + +trigger: none +pr: none + +stages: +- stage: Build + displayName: Build binaries + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-build.yml + +- stage: Sign + displayName: Sign binaries + dependsOn: Build + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-sign.yml + +- stage: Layout + displayName: Generate layouts + dependsOn: Sign + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-layout-full.yml + - template: windows-release/stage-layout-embed.yml + - template: windows-release/stage-layout-nuget.yml + +- stage: Pack + dependsOn: Layout + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-pack-nuget.yml + +- stage: Test + dependsOn: Pack + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-test-embed.yml + - template: windows-release/stage-test-nuget.yml + +- stage: Layout_MSIX + displayName: Generate MSIX layouts + dependsOn: Sign + condition: and(succeeded(), and(eq(variables['DoMSIX'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-layout-msix.yml + +- stage: Pack_MSIX + displayName: Package MSIX + dependsOn: Layout_MSIX + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-pack-msix.yml + +- stage: Build_MSI + displayName: Build MSI installer + dependsOn: Sign + condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-msi.yml + +- stage: Test_MSI + displayName: Test MSI installer + dependsOn: Build_MSI + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-test-msi.yml + +- stage: PublishPyDotOrg + displayName: Publish to python.org + dependsOn: ['Test_MSI', 'Test'] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-publish-pythonorg.yml + +- stage: PublishNuget + displayName: Publish to nuget.org + dependsOn: Test + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-publish-nugetorg.yml + +- stage: PublishStore + displayName: Publish to Store + dependsOn: Pack_MSIX + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-publish-store.yml + + +- stage: PublishExistingPyDotOrg + displayName: Publish existing build to python.org + dependsOn: [] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-publish-pythonorg.yml + +- stage: PublishExistingNuget + displayName: Publish existing build to nuget.org + dependsOn: [] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-publish-nugetorg.yml + +- stage: PublishExistingStore + displayName: Publish existing build to Store + dependsOn: [] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-publish-store.yml diff --git a/.azure-pipelines/windows-release/build-steps.yml b/.azure-pipelines/windows-release/build-steps.yml new file mode 100644 index 00000000000000..d4563cd0d722c1 --- /dev/null +++ b/.azure-pipelines/windows-release/build-steps.yml @@ -0,0 +1,83 @@ +parameters: + ShouldPGO: false + +steps: +- template: ./checkout.yml + +- powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)" + Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)" + Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)" + Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)" + Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)-$(Name)" + displayName: 'Extract version numbers' + +- ${{ if eq(parameters.ShouldPGO, 'false') }}: + - powershell: | + $env:SigningCertificate = $null + .\PCbuild\build.bat -v -p $(Platform) -c $(Configuration) + displayName: 'Run build' + env: + IncludeUwp: true + Py_OutDir: '$(Build.BinariesDirectory)\bin' + +- ${{ if eq(parameters.ShouldPGO, 'true') }}: + - powershell: | + $env:SigningCertificate = $null + .\PCbuild\build.bat -v -p $(Platform) --pgo + displayName: 'Run build with PGO' + env: + IncludeUwp: true + Py_OutDir: '$(Build.BinariesDirectory)\bin' + +- powershell: | + $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10 + $tool = (gci -r "$kitroot\Bin\*\x64\signtool.exe" | sort FullName -Desc | select -First 1) + if (-not $tool) { + throw "SDK is not available" + } + Write-Host "##vso[task.prependpath]$($tool.Directory)" + displayName: 'Add WinSDK tools to path' + +- powershell: | + $env:SigningCertificate = $null + .\python.bat PC\layout -vv -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default + makecat "${env:CAT}.cdf" + del "${env:CAT}.cdf" + if (-not (Test-Path "${env:CAT}.cat")) { + throw "Failed to build catalog file" + } + displayName: 'Generate catalog' + env: + CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python + +- task: PublishPipelineArtifact@0 + displayName: 'Publish binaries' + condition: and(succeeded(), not(and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate']))) + inputs: + targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)' + artifactName: bin_$(Name) + +- task: PublishPipelineArtifact@0 + displayName: 'Publish binaries for signing' + condition: and(succeeded(), and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate'])) + inputs: + targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)' + artifactName: unsigned_bin_$(Name) + +- task: CopyFiles@2 + displayName: 'Layout Artifact: symbols' + inputs: + sourceFolder: $(Build.BinariesDirectory)\bin\$(Arch) + targetFolder: $(Build.ArtifactStagingDirectory)\symbols\$(Name) + flatten: true + contents: | + **\*.pdb + +- task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: symbols' + inputs: + PathToPublish: '$(Build.ArtifactStagingDirectory)\symbols' + ArtifactName: symbols diff --git a/.azure-pipelines/windows-release/checkout.yml b/.azure-pipelines/windows-release/checkout.yml new file mode 100644 index 00000000000000..d42d55fff08dda --- /dev/null +++ b/.azure-pipelines/windows-release/checkout.yml @@ -0,0 +1,21 @@ +parameters: + depth: 3 + +steps: +- checkout: none + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch https://github.com/$(GitRemote)/cpython.git . + displayName: 'git clone ($(GitRemote)/$(SourceTag))' + condition: and(succeeded(), and(variables['GitRemote'], variables['SourceTag'])) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch $(Build.Repository.Uri) . + displayName: 'git clone (/$(SourceTag))' + condition: and(succeeded(), and(not(variables['GitRemote']), variables['SourceTag'])) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch https://github.com/$(GitRemote)/cpython.git . + displayName: 'git clone ($(GitRemote)/)' + condition: and(succeeded(), and(variables['GitRemote'], not(variables['SourceTag']))) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch $(Build.Repository.Uri) . + displayName: 'git clone' + condition: and(succeeded(), and(not(variables['GitRemote']), not(variables['SourceTag']))) diff --git a/.azure-pipelines/windows-release/find-sdk.yml b/.azure-pipelines/windows-release/find-sdk.yml new file mode 100644 index 00000000000000..e4de78555b3f66 --- /dev/null +++ b/.azure-pipelines/windows-release/find-sdk.yml @@ -0,0 +1,17 @@ +# Locate the Windows SDK and add its binaries directory to PATH +# +# `toolname` can be overridden to use a different marker file. + +parameters: + toolname: signtool.exe + +steps: + - powershell: | + $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10 + $tool = (gci -r "$kitroot\Bin\*\${{ parameters.toolname }}" | sort FullName -Desc | select -First 1) + if (-not $tool) { + throw "SDK is not available" + } + Write-Host "##vso[task.prependpath]$($tool.Directory)" + Write-Host "Adding $($tool.Directory) to PATH" + displayName: 'Add WinSDK tools to path' diff --git a/.azure-pipelines/windows-release/gpg-sign.yml b/.azure-pipelines/windows-release/gpg-sign.yml new file mode 100644 index 00000000000000..0855af8d703df9 --- /dev/null +++ b/.azure-pipelines/windows-release/gpg-sign.yml @@ -0,0 +1,28 @@ +parameters: + GPGKeyFile: $(GPGKey) + GPGPassphrase: $(GPGPassphrase) + Files: '*' + WorkingDirectory: $(Build.BinariesDirectory) + +steps: +- task: DownloadSecureFile@1 + name: gpgkey + inputs: + secureFile: ${{ parameters.GPGKeyFile }} + displayName: 'Download GPG key' + +- powershell: | + git clone https://github.com/python/cpython-bin-deps --branch gpg --single-branch --depth 1 --progress -v "gpg" + gpg/gpg2.exe --import "$(gpgkey.secureFilePath)" + (gci -File ${{ parameters.Files }}).FullName | %{ + gpg/gpg2.exe -ba --batch --passphrase ${{ parameters.GPGPassphrase }} $_ + "Made signature for $_" + } + displayName: 'Generate GPG signatures' + workingDirectory: ${{ parameters.WorkingDirectory }} + +- powershell: | + $p = gps "gpg-agent" -EA 0 + if ($p) { $p.Kill() } + displayName: 'Kill GPG agent' + condition: true diff --git a/.azure-pipelines/windows-release/layout-command.yml b/.azure-pipelines/windows-release/layout-command.yml new file mode 100644 index 00000000000000..2dcd6ed26ca3a8 --- /dev/null +++ b/.azure-pipelines/windows-release/layout-command.yml @@ -0,0 +1,15 @@ +steps: +- powershell: > + Write-Host ( + '##vso[task.setvariable variable=LayoutCmd]& + "{0}\bin\python.exe" + "{1}\PC\layout" + -vv + --source "{1}" + --build "{0}\bin" + --temp "{0}\layout-temp" + --include-cat "{0}\bin\python.cat" + --doc-build "{0}\doc"' + -f ("$(Build.BinariesDirectory)", "$(Build.SourcesDirectory)") + ) + displayName: 'Set LayoutCmd' diff --git a/.azure-pipelines/windows-release/mingw-lib.yml b/.azure-pipelines/windows-release/mingw-lib.yml new file mode 100644 index 00000000000000..30f7d34fa61d23 --- /dev/null +++ b/.azure-pipelines/windows-release/mingw-lib.yml @@ -0,0 +1,13 @@ +parameters: + DllToolOpt: -m i386:x86-64 + #DllToolOpt: -m i386 --as-flags=--32 + +steps: +- powershell: | + git clone https://github.com/python/cpython-bin-deps --branch binutils --single-branch --depth 1 --progress -v "binutils" + gci "bin\$(Arch)\python*.dll" | %{ + & "binutils\gendef.exe" $_ | Out-File -Encoding ascii tmp.def + & "binutils\dlltool.exe" --dllname $($_.BaseName).dll --def tmp.def --output-lib "$($_.Directory)\lib$($_.BaseName).a" ${{ parameters.DllToolOpt }} + } + displayName: 'Generate MinGW import library' + workingDirectory: $(Build.BinariesDirectory) diff --git a/.azure-pipelines/windows-release/msi-steps.yml b/.azure-pipelines/windows-release/msi-steps.yml new file mode 100644 index 00000000000000..f7bff162f8e027 --- /dev/null +++ b/.azure-pipelines/windows-release/msi-steps.yml @@ -0,0 +1,128 @@ +steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: doc' + inputs: + artifactName: doc + targetPath: $(Build.BinariesDirectory)\doc + + - task: CopyFiles@2 + displayName: 'Merge documentation files' + inputs: + sourceFolder: $(Build.BinariesDirectory)\doc + targetFolder: $(Build.SourcesDirectory)\Doc\build + contents: | + htmlhelp\*.chm + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_win32' + inputs: + artifactName: bin_win32 + targetPath: $(Build.BinariesDirectory)\win32 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_win32_d' + inputs: + artifactName: bin_win32_d + targetPath: $(Build.BinariesDirectory)\win32 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_amd64' + inputs: + artifactName: bin_amd64 + targetPath: $(Build.BinariesDirectory)\amd64 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_amd64_d' + inputs: + artifactName: bin_amd64_d + targetPath: $(Build.BinariesDirectory)\amd64 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_win32' + inputs: + artifactName: tcltk_lib_win32 + targetPath: $(Build.BinariesDirectory)\tcltk_lib_win32 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_amd64' + inputs: + artifactName: tcltk_lib_amd64 + targetPath: $(Build.BinariesDirectory)\tcltk_lib_amd64 + + - powershell: | + copy $(Build.BinariesDirectory)\amd64\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force + displayName: 'Copy signed files into sources' + + - script: | + call Tools\msi\get_externals.bat + call PCbuild\find_python.bat + echo ##vso[task.setvariable variable=PYTHON]%PYTHON% + call PCbuild/find_msbuild.bat + echo ##vso[task.setvariable variable=MSBUILD]%MSBUILD% + displayName: 'Get external dependencies' + + - script: | + %PYTHON% -m pip install blurb + %PYTHON% -m blurb merge -f Misc\NEWS + displayName: 'Merge NEWS file' + + - script: | + %MSBUILD% Tools\msi\launcher\launcher.wixproj + displayName: 'Build launcher installer' + env: + Platform: x86 + Py_OutDir: $(Build.BinariesDirectory) + + - script: | + %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true + %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false + displayName: 'Build win32 installer' + env: + Platform: x86 + Py_OutDir: $(Build.BinariesDirectory) + PYTHON: $(Build.BinariesDirectory)\win32\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_win32 + BuildForRelease: true + SuppressMinGWLib: true + + - script: | + %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true + %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false + displayName: 'Build amd64 installer' + env: + Platform: x64 + Py_OutDir: $(Build.BinariesDirectory) + PYTHON: $(Build.BinariesDirectory)\amd64\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_amd64 + BuildForRelease: true + SuppressMinGWLib: true + + - task: CopyFiles@2 + displayName: 'Assemble artifact: msi (1/2)' + inputs: + sourceFolder: $(Build.BinariesDirectory)\win32\en-us + targetFolder: $(Build.ArtifactStagingDirectory)\msi\win32 + contents: | + *.msi + *.cab + *.exe + + - task: CopyFiles@2 + displayName: 'Assemble artifact: msi (2/2)' + inputs: + sourceFolder: $(Build.BinariesDirectory)\amd64\en-us + targetFolder: $(Build.ArtifactStagingDirectory)\msi\amd64 + contents: | + *.msi + *.cab + *.exe + + - task: PublishPipelineArtifact@0 + displayName: 'Publish MSI' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\msi' + artifactName: msi diff --git a/.azure-pipelines/windows-release/stage-build.yml b/.azure-pipelines/windows-release/stage-build.yml new file mode 100644 index 00000000000000..c98576ef9705c0 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-build.yml @@ -0,0 +1,160 @@ +jobs: +- job: Build_Docs + displayName: Docs build + pool: + name: 'Windows Release' + #vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - script: Doc\make.bat html + displayName: 'Build HTML docs' + env: + BUILDDIR: $(Build.BinariesDirectory)\Doc + + #- powershell: iwr "https://www.python.org/ftp/python/3.7.3/python373.chm" -OutFile "$(Build.BinariesDirectory)\python390a0.chm" + # displayName: 'Cheat at building CHM docs' + + - script: Doc\make.bat htmlhelp + displayName: 'Build CHM docs' + env: + BUILDDIR: $(Build.BinariesDirectory)\Doc + + - task: CopyFiles@2 + displayName: 'Assemble artifact: Doc' + inputs: + sourceFolder: $(Build.BinariesDirectory)\Doc + targetFolder: $(Build.ArtifactStagingDirectory)\Doc + contents: | + html\**\* + htmlhelp\*.chm + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: doc' + inputs: + targetPath: $(Build.ArtifactStagingDirectory)\Doc + artifactName: doc + +- job: Build_Python + displayName: Python build + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Arch: win32 + Platform: x86 + Configuration: Release + win32_d: + Name: win32_d + Arch: win32 + Platform: x86 + Configuration: Debug + amd64_d: + Name: amd64_d + Arch: amd64 + Platform: x64 + Configuration: Debug + + steps: + - template: ./build-steps.yml + +- job: Build_Python_NonPGO + displayName: Python non-PGO build + condition: and(succeeded(), ne(variables['DoPGO'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Arch: amd64 + Platform: x64 + Configuration: Release + + steps: + - template: ./build-steps.yml + + +- job: Build_Python_PGO + displayName: Python PGO build + condition: and(succeeded(), eq(variables['DoPGO'], 'true')) + + # Allow up to five hours for PGO + timeoutInMinutes: 300 + + pool: + name: 'Windows Release' + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Arch: amd64 + Platform: x64 + Configuration: Release + + steps: + - template: ./build-steps.yml + parameters: + ShouldPGO: true + + +- job: TclTk_Lib + displayName: Publish Tcl/Tk Library + + pool: + vmName: windows-latest + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - script: PCbuild\get_externals.bat --no-openssl --no-libffi + displayName: 'Get external dependencies' + + - task: MSBuild@1 + displayName: 'Copy Tcl/Tk lib for publish' + inputs: + solution: PCbuild\tcltk.props + platform: x86 + msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_win32" + + - task: MSBuild@1 + displayName: 'Copy Tcl/Tk lib for publish' + inputs: + solution: PCbuild\tcltk.props + platform: x64 + msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_amd64" + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: tcltk_lib_win32' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\tcl_win32' + artifactName: tcltk_lib_win32 + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: tcltk_lib_amd64' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\tcl_amd64' + artifactName: tcltk_lib_amd64 diff --git a/.azure-pipelines/windows-release/stage-layout-embed.yml b/.azure-pipelines/windows-release/stage-layout-embed.yml new file mode 100644 index 00000000000000..09857ff676b355 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-embed.yml @@ -0,0 +1,56 @@ +jobs: +- job: Make_Embed_Layout + displayName: Make embeddable layout + condition: and(succeeded(), eq(variables['DoEmbed'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - template: ./layout-command.yml + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + displayName: 'Extract version numbers' + + - powershell: > + $(LayoutCmd) + --copy "$(Build.ArtifactStagingDirectory)\layout" + --zip "$(Build.ArtifactStagingDirectory)\embed\python-$(VersionText)-embed-$(Name).zip" + --preset-embed + displayName: 'Generate embeddable layout' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_embed_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\layout' + artifactName: layout_embed_$(Name) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: embed' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\embed' + ArtifactName: embed diff --git a/.azure-pipelines/windows-release/stage-layout-full.yml b/.azure-pipelines/windows-release/stage-layout-full.yml new file mode 100644 index 00000000000000..12c347239013c1 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-full.yml @@ -0,0 +1,66 @@ +jobs: +- job: Make_Layouts + displayName: Make layouts + condition: and(succeeded(), eq(variables['DoLayout'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)_d' + inputs: + artifactName: bin_$(Name)_d + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: doc' + inputs: + artifactName: doc + targetPath: $(Build.BinariesDirectory)\doc + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_$(Name)' + inputs: + artifactName: tcltk_lib_$(Name) + targetPath: $(Build.BinariesDirectory)\tcltk_lib + + - powershell: | + copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force + displayName: 'Copy signed files into sources' + + - template: ./layout-command.yml + + - powershell: | + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\layout" --preset-default + displayName: 'Generate full layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_full_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\layout' + artifactName: layout_full_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-msix.yml b/.azure-pipelines/windows-release/stage-layout-msix.yml new file mode 100644 index 00000000000000..ba86392f3ec699 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-msix.yml @@ -0,0 +1,88 @@ +jobs: +- job: Make_MSIX_Layout + displayName: Make MSIX layout + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + #win32: + # Name: win32 + # Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + # PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)_d' + inputs: + artifactName: bin_$(Name)_d + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_$(Name)' + inputs: + artifactName: tcltk_lib_$(Name) + targetPath: $(Build.BinariesDirectory)\tcltk_lib + + - powershell: | + copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force + displayName: 'Copy signed files into sources' + + - template: ./layout-command.yml + + - powershell: | + Remove-Item "$(Build.ArtifactStagingDirectory)\appx-store" -Recurse -Force -EA 0 + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx-store" --preset-appx --precompile + displayName: 'Generate store APPX layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_appxstore_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\appx-store' + artifactName: layout_appxstore_$(Name) + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: cert' + condition: and(succeeded(), variables['SigningCertificate']) + inputs: + artifactName: cert + targetPath: $(Build.BinariesDirectory)\cert + + - powershell: | + $info = (gc "$(Build.BinariesDirectory)\cert\certinfo.json" | ConvertFrom-JSON) + Write-Host "Side-loadable APPX must be signed with '$($info.Subject)'" + Write-Host "##vso[task.setvariable variable=APPX_DATA_PUBLISHER]$($info.Subject)" + Write-Host "##vso[task.setvariable variable=APPX_DATA_SHA256]$($info.SHA256)" + displayName: 'Override signing parameters' + condition: and(succeeded(), variables['SigningCertificate']) + + - powershell: | + Remove-Item "$(Build.ArtifactStagingDirectory)\appx" -Recurse -Force -EA 0 + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx" --preset-appx --precompile --include-symbols --include-tests + displayName: 'Generate sideloading APPX layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_appx_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\appx' + artifactName: layout_appx_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-nuget.yml b/.azure-pipelines/windows-release/stage-layout-nuget.yml new file mode 100644 index 00000000000000..7954c4547f50ab --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-nuget.yml @@ -0,0 +1,48 @@ +jobs: +- job: Make_Nuget_Layout + displayName: Make Nuget layout + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - powershell: | + copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force + displayName: 'Copy signed files into sources' + + - template: ./layout-command.yml + + - powershell: | + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\nuget" --preset-nuget + displayName: 'Generate nuget layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\bin_$(Name)\tcl\tcl8 + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_nuget_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\nuget' + artifactName: layout_nuget_$(Name) diff --git a/.azure-pipelines/windows-release/stage-msi.yml b/.azure-pipelines/windows-release/stage-msi.yml new file mode 100644 index 00000000000000..7afc816a0c6e9c --- /dev/null +++ b/.azure-pipelines/windows-release/stage-msi.yml @@ -0,0 +1,36 @@ +jobs: +- job: Make_MSI + displayName: Make MSI + condition: and(succeeded(), not(variables['SigningCertificate'])) + + pool: + vmName: win2016-vs2017 + + variables: + ReleaseUri: http://www.python.org/{arch} + DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} + Py_OutDir: $(Build.BinariesDirectory) + + workspace: + clean: all + + steps: + - template: msi-steps.yml + +- job: Make_Signed_MSI + displayName: Make signed MSI + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + variables: + ReleaseUri: http://www.python.org/{arch} + DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} + Py_OutDir: $(Build.BinariesDirectory) + + workspace: + clean: all + + steps: + - template: msi-steps.yml diff --git a/.azure-pipelines/windows-release/stage-pack-msix.yml b/.azure-pipelines/windows-release/stage-pack-msix.yml new file mode 100644 index 00000000000000..eebc63fb8809bc --- /dev/null +++ b/.azure-pipelines/windows-release/stage-pack-msix.yml @@ -0,0 +1,127 @@ +jobs: +- job: Pack_MSIX + displayName: Pack MSIX bundles + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Artifact: appx + Suffix: + ShouldSign: true + amd64_store: + Name: amd64 + Artifact: appxstore + Suffix: -store + Upload: true + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: layout_$(Artifact)_$(Name)' + inputs: + artifactName: layout_$(Artifact)_$(Name) + targetPath: $(Build.BinariesDirectory)\layout + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: symbols' + inputs: + artifactName: symbols + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)" + Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)" + Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)" + Write-Host "##vso[task.setvariable variable=Filename]python-$($d.PythonVersion)-$(Name)$(Suffix)" + displayName: 'Extract version numbers' + + - powershell: | + ./Tools/msi/make_appx.ps1 -layout "$(Build.BinariesDirectory)\layout" -msix "$(Build.ArtifactStagingDirectory)\msix\$(Filename).msix" + displayName: 'Build msix' + + - powershell: | + 7z a -tzip "$(Build.ArtifactStagingDirectory)\msix\$(Filename).appxsym" *.pdb + displayName: 'Build appxsym' + workingDirectory: $(Build.BinariesDirectory)\symbols\$(Name) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + condition: and(succeeded(), or(ne(variables['ShouldSign'], 'true'), not(variables['SigningCertificate']))) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix' + ArtifactName: msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + condition: and(succeeded(), and(eq(variables['ShouldSign'], 'true'), variables['SigningCertificate'])) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix' + ArtifactName: unsigned_msix + + - powershell: | + 7z a -tzip "$(Build.ArtifactStagingDirectory)\msixupload\$(Filename).msixupload" * + displayName: 'Build msixupload' + condition: and(succeeded(), eq(variables['Upload'], 'true')) + workingDirectory: $(Build.ArtifactStagingDirectory)\msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIXUpload' + condition: and(succeeded(), eq(variables['Upload'], 'true')) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msixupload' + ArtifactName: msixupload + + +- job: Sign_MSIX + displayName: Sign side-loadable MSIX bundles + dependsOn: + - Pack_MSIX + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + workspace: + clean: all + + steps: + - checkout: none + - template: ./find-sdk.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Artifact: unsigned_msix' + inputs: + artifactName: unsigned_msix + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $failed = $true + foreach ($retry in 1..3) { + signtool sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "$(SigningDescription)" (gi *.msix) + if ($?) { + $failed = $false + break + } + sleep 1 + } + if ($failed) { + throw "Failed to sign MSIX" + } + displayName: 'Sign MSIX' + workingDirectory: $(Build.BinariesDirectory)\unsigned_msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + inputs: + PathtoPublish: '$(Build.BinariesDirectory)\unsigned_msix' + ArtifactName: msix diff --git a/.azure-pipelines/windows-release/stage-pack-nuget.yml b/.azure-pipelines/windows-release/stage-pack-nuget.yml new file mode 100644 index 00000000000000..f59bbe9b39a8d7 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-pack-nuget.yml @@ -0,0 +1,41 @@ +jobs: +- job: Pack_Nuget + displayName: Pack Nuget bundles + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + win32: + Name: win32 + + steps: + - checkout: none + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: layout_nuget_$(Name)' + inputs: + artifactName: layout_nuget_$(Name) + targetPath: $(Build.BinariesDirectory)\layout + + - task: NugetToolInstaller@0 + displayName: 'Install Nuget' + inputs: + versionSpec: '>=5.0' + + - powershell: | + nuget pack "$(Build.BinariesDirectory)\layout\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive + displayName: 'Create nuget package' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: nuget' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: nuget diff --git a/.azure-pipelines/windows-release/stage-publish-nugetorg.yml b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml new file mode 100644 index 00000000000000..570cdb3ec57f0d --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml @@ -0,0 +1,41 @@ +jobs: +- job: Publish_Nuget + displayName: Publish Nuget packages + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + buildType: specific + project: cpython + pipeline: Windows-Release + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + - task: NuGetCommand@2 + displayName: Push packages + condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) + inputs: + command: push + packagesToPush: '$(Build.BinariesDirectory)\nuget\*.nupkg' + nuGetFeedType: external + publishFeedCredentials: 'Python on Nuget' diff --git a/.azure-pipelines/windows-release/stage-publish-pythonorg.yml b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml new file mode 100644 index 00000000000000..2dd354a8c276f6 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml @@ -0,0 +1,154 @@ +jobs: +- job: Publish_Python + displayName: Publish python.org packages + condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), eq(variables['DoEmbed'], 'true'))) + + pool: + #vmName: win2016-vs2017 + name: 'Windows Release' + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6 or later' + inputs: + versionSpec: '>=3.6' + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: Doc' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: Doc + targetPath: $(Build.BinariesDirectory)\Doc + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: msi' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: msi + targetPath: $(Build.BinariesDirectory)\msi + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: embed' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact from $(BuildToPublish): Doc' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: Doc + targetPath: $(Build.BinariesDirectory)\Doc + buildType: specific + project: cpython + pipeline: 21 + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact from $(BuildToPublish): msi' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: msi + targetPath: $(Build.BinariesDirectory)\msi + buildType: specific + project: cpython + pipeline: 21 + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact from $(BuildToPublish): embed' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + buildType: specific + project: cpython + pipeline: Windows-Release + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + + - template: ./gpg-sign.yml + parameters: + GPGKeyFile: 'python-signing.key' + Files: 'doc\htmlhelp\*.chm, msi\*\*, embed\*.zip' + + - powershell: > + $(Build.SourcesDirectory)\Tools\msi\uploadrelease.ps1 + -build msi + -user $(PyDotOrgUsername) + -server $(PyDotOrgServer) + -doc_htmlhelp doc\htmlhelp + -embed embed + -skippurge + -skiptest + -skiphash + condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Upload files to python.org' + + - powershell: > + python + "$(Build.SourcesDirectory)\Tools\msi\purge.py" + (gci msi\*\python-*.exe | %{ $_.Name -replace 'python-(.+?)(-|\.exe).+', '$1' } | select -First 1) + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Purge CDN' + + - powershell: | + $failures = 0 + gci "msi\*\*-webinstall.exe" -File | %{ + $d = mkdir "tests\$($_.BaseName)" -Force + gci $d -r -File | del + $ic = copy $_ $d -PassThru + "Checking layout for $($ic.Name)" + Start-Process -wait $ic "/passive", "/layout", "$d\layout", "/log", "$d\log\install.log" + if (-not $?) { + Write-Error "Failed to validate layout of $($inst.Name)" + $failures += 1 + } + } + if ($failures) { + Write-Error "Failed to validate $failures installers" + exit 1 + } + #condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Test layouts' + + - powershell: | + $hashes = gci doc\htmlhelp\python*.chm, msi\*\*.exe, embed\*.zip | ` + Sort-Object Name | ` + Format-Table Name, @{ + Label="MD5"; + Expression={(Get-FileHash $_ -Algorithm MD5).Hash} + }, Length -AutoSize | ` + Out-String -Width 4096 + $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force + $hashes | Out-File "$d\hashes.txt" -Encoding ascii + $hashes + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Generate hashes' + + - powershell: | + "Copying:" + (gci msi\*\python*.asc, doc\htmlhelp\*.asc, embed\*.asc).FullName + $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force + move msi\*\python*.asc, doc\htmlhelp\*.asc, embed\*.asc $d -Force + gci msi -Directory | %{ move "msi\$_\*.asc" (mkdir "$d\$_" -Force) } + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Copy GPG signatures for build' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: hashes' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\hashes' + artifactName: hashes diff --git a/.azure-pipelines/windows-release/stage-publish-store.yml b/.azure-pipelines/windows-release/stage-publish-store.yml new file mode 100644 index 00000000000000..b22147b1ab45e7 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-store.yml @@ -0,0 +1,35 @@ +jobs: +- job: Publish_Store + displayName: Publish Store packages + condition: and(succeeded(), eq(variables['DoMSIX'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: msixupload' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: msixupload + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: msixupload' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: msixupload + downloadPath: $(Build.BinariesDirectory) + buildType: specific + project: cpython + pipeline: Windows-Release + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + # TODO: eq(variables['SigningCertificate'], variables['__RealSigningCertificate']) + # If we are not real-signed, DO NOT PUBLISH diff --git a/.azure-pipelines/windows-release/stage-sign.yml b/.azure-pipelines/windows-release/stage-sign.yml new file mode 100644 index 00000000000000..2307c6c9c8f97d --- /dev/null +++ b/.azure-pipelines/windows-release/stage-sign.yml @@ -0,0 +1,122 @@ +parameters: + Include: '*.exe, *.dll, *.pyd, *.cat, *.ps1' + Exclude: 'vcruntime*, libffi*, libcrypto*, libssl*' + +jobs: +- job: Sign_Python + displayName: Sign Python binaries + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + amd64: + Name: amd64 + + steps: + - template: ./checkout.yml + - template: ./find-sdk.yml + + - powershell: | + Write-Host "##vso[build.addbuildtag]signed" + displayName: 'Add build tags' + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: unsigned_bin_$(Name)' + inputs: + artifactName: unsigned_bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - powershell: | + copy "$(Build.SourcesDirectory)\Lib\venv\scripts\common\Activate.ps1" . + displayName: 'Copy files from source' + workingDirectory: $(Build.BinariesDirectory)\bin + + - powershell: | + $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }}) + signtool sign /a /n "$(SigningCertificate)" /fd sha256 /d "$(SigningDescription)" $files + displayName: 'Sign binaries' + workingDirectory: $(Build.BinariesDirectory)\bin + + - powershell: | + $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }}) + $failed = $true + foreach ($retry in 1..10) { + signtool timestamp /t http://timestamp.verisign.com/scripts/timestamp.dll $files + if ($?) { + $failed = $false + break + } + sleep 5 + } + if ($failed) { + Write-Host "##vso[task.logissue type=error]Failed to timestamp files" + } + displayName: 'Timestamp binaries' + workingDirectory: $(Build.BinariesDirectory)\bin + continueOnError: true + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: bin_$(Name)' + inputs: + targetPath: '$(Build.BinariesDirectory)\bin' + artifactName: bin_$(Name) + + +- job: Dump_CertInfo + displayName: Capture certificate info + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + steps: + - checkout: none + + - powershell: | + $m = 'CN=$(SigningCertificate)' + $c = ((gci Cert:\CurrentUser\My), (gci Cert:\LocalMachine\My)) | %{ $_ } | ` + ?{ $_.Subject -match $m } | ` + select -First 1 + if (-not $c) { + Write-Host "Failed to find certificate for $(SigningCertificate)" + exit + } + $d = mkdir "$(Build.BinariesDirectory)\tmp" -Force + $cf = "$d\cert.cer" + [IO.File]::WriteAllBytes($cf, $c.Export("Cer")) + $csha = (certutil -dump $cf | sls "Cert Hash\(sha256\): (.+)").Matches.Groups[1].Value + + $info = @{ Subject=$c.Subject; SHA256=$csha; } + + $d = mkdir "$(Build.BinariesDirectory)\cert" -Force + $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json" + displayName: "Extract certificate info" + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: cert' + inputs: + targetPath: '$(Build.BinariesDirectory)\cert' + artifactName: cert + + +- job: Mark_Unsigned + displayName: Tag unsigned build + condition: and(succeeded(), not(variables['SigningCertificate'])) + + pool: + vmName: win2016-vs2017 + + steps: + - checkout: none + + - powershell: | + Write-Host "##vso[build.addbuildtag]unsigned" + displayName: 'Add build tag' diff --git a/.azure-pipelines/windows-release/stage-test-embed.yml b/.azure-pipelines/windows-release/stage-test-embed.yml new file mode 100644 index 00000000000000..b33176266a20da --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-embed.yml @@ -0,0 +1,41 @@ +jobs: +- job: Test_Embed + displayName: Test Embed + condition: and(succeeded(), eq(variables['DoEmbed'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + amd64: + Name: amd64 + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: embed' + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\embed\python*embed-$(Name).zip" + Expand-Archive -Path $p -DestinationPath "$(Build.BinariesDirectory)\Python" + $p = gi "$(Build.BinariesDirectory)\Python\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Install Python and add to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' diff --git a/.azure-pipelines/windows-release/stage-test-msi.yml b/.azure-pipelines/windows-release/stage-test-msi.yml new file mode 100644 index 00000000000000..27b0c96987a7a0 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-msi.yml @@ -0,0 +1,108 @@ +jobs: +- job: Test_MSI + displayName: Test MSI + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32_User: + ExeMatch: 'python-[\dabrc.]+-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\win32_User + InstallAllUsers: 0 + win32_Machine: + ExeMatch: 'python-[\dabrc.]+-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\win32_Machine + InstallAllUsers: 1 + amd64_User: + ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_User + InstallAllUsers: 0 + amd64_Machine: + ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_Machine + InstallAllUsers: 1 + + steps: + - checkout: none + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: msi' + inputs: + artifactName: msi + targetPath: $(Build.BinariesDirectory)\msi + + - powershell: | + $p = (gci -r *.exe | ?{ $_.Name -match '$(ExeMatch)' } | select -First 1) + Write-Host "##vso[task.setvariable variable=SetupExe]$($p.FullName)" + Write-Host "##vso[task.setvariable variable=SetupExeName]$($p.Name)" + displayName: 'Find installer executable' + workingDirectory: $(Build.BinariesDirectory)\msi + + - script: > + "$(SetupExe)" + /passive + /log "$(Logs)\install\log.txt" + TargetDir="$(Build.BinariesDirectory)\Python" + Include_debug=1 + Include_symbols=1 + InstallAllUsers=$(InstallAllUsers) + displayName: 'Install Python' + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\Python\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Add test Python to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' + + - powershell: | + gci -r "${env:PROGRAMDATA}\Microsoft\Windows\Start Menu\Programs\Python*" + displayName: 'Capture per-machine Start Menu items' + - powershell: | + gci -r "${env:APPDATA}\Microsoft\Windows\Start Menu\Programs\Python*" + displayName: 'Capture per-user Start Menu items' + + - powershell: | + gci -r "HKLM:\Software\WOW6432Node\Python" + displayName: 'Capture per-machine 32-bit registry' + - powershell: | + gci -r "HKLM:\Software\Python" + displayName: 'Capture per-machine native registry' + - powershell: | + gci -r "HKCU:\Software\Python" + displayName: 'Capture current-user registry' + + - script: | + python -m pip install "azure<0.10" + python -m pip uninstall -y azure python-dateutil six + displayName: 'Test (un)install package' + + - script: | + python -m test -uall -v test_ttk_guionly test_tk test_idle + displayName: 'Test Tkinter and Idle' + + - script: > + "$(SetupExe)" + /passive + /uninstall + /log "$(Logs)\uninstall\log.txt" + displayName: 'Uninstall Python' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: logs' + condition: true + continueOnError: true + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\logs' + ArtifactName: msi_testlogs diff --git a/.azure-pipelines/windows-release/stage-test-nuget.yml b/.azure-pipelines/windows-release/stage-test-nuget.yml new file mode 100644 index 00000000000000..1f8b601d0d023b --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-nuget.yml @@ -0,0 +1,58 @@ +jobs: +- job: Test_Nuget + displayName: Test Nuget + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Package: pythonx86 + amd64: + Package: python + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + + - task: NugetToolInstaller@0 + inputs: + versionSpec: '>= 5' + + - powershell: > + nuget install + $(Package) + -Source "$(Build.BinariesDirectory)\nuget" + -OutputDirectory "$(Build.BinariesDirectory)\install" + -Prerelease + -ExcludeVersion + -NonInteractive + displayName: 'Install Python' + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\install\$(Package)\tools\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Add test Python to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' + + - script: | + python -m pip install "azure<0.10" + python -m pip uninstall -y azure python-dateutil six + displayName: 'Test (un)install package' diff --git a/.gitattributes b/.gitattributes index c9a54fbd472ecc..bec16a08152ebb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -54,7 +54,7 @@ Python/Python-ast.c linguist-generated=true Include/opcode.h linguist-generated=true Python/opcode_targets.h linguist-generated=true Objects/typeslots.inc linguist-generated=true -Modules/unicodedata_db.h linguist-generated=true +*_db.h linguist-generated=true Doc/library/token-list.inc linguist-generated=true Include/token.h linguist-generated=true Lib/token.py linguist-generated=true diff --git a/.gitignore b/.gitignore index 9445ef1e2c5252..648f07e765b122 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,8 @@ -# added for local development -.buildaix/ -Modules/python.exp -buildaix/ -installp/ -.gitignore - # Two-trick pony for OSX and other case insensitive file systems: # Ignore ./python binary on Unix but still look into ./Python/ directory. /python !/Python/ + *.cover *.iml *.o @@ -33,12 +27,12 @@ Include/pydtrace_probes.h Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* -Makefile +!Lib/test/data/README +/Makefile Makefile.pre Misc/python.pc Misc/python-embed.pc Misc/python-config.sh -Modules/Setup Modules/Setup.config Modules/Setup.local Modules/config.c @@ -84,6 +78,7 @@ config.log config.status config.status.lineno core +!Tools/msi/core/ db_home .hg/ .idea/ @@ -94,7 +89,7 @@ libpython*.dylib libpython*.dll platform pybuilddir.txt -pyconfig.h +/pyconfig.h python-config python-config.py python.bat diff --git a/.travis.yml b/.travis.yml index 02de997750abc8..1d470728a536a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,7 @@ env: - OPENSSL=1.1.1c - OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}" - PATH="${OPENSSL_DIR}/bin:$PATH" - # Use -O3 because we don't use debugger on Travis-CI - - CFLAGS="-I${OPENSSL_DIR}/include -O3" + - CFLAGS="-I${OPENSSL_DIR}/include" - LDFLAGS="-L${OPENSSL_DIR}/lib" # Set rpath with env var instead of -Wl,-rpath linker flag # OpenSSL ignores LDFLAGS when linking bin/openssl @@ -43,6 +42,7 @@ matrix: addons: apt: packages: + - gdb - xvfb - name: "Documentation build" os: linux @@ -166,7 +166,8 @@ install: # Travis provides only 2 cores, so don't overdo the parallelism and waste memory. before_script: - - ./configure --with-pydebug + # -Og is much faster than -O0 + - CFLAGS="${CFLAGS} -Og" ./configure --with-pydebug - make -j4 regen-all - changes=`git status --porcelain` - | diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 1e044ad2033d87..a17f04d26fa40b 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -80,7 +80,7 @@ taken on the bug. Article which goes into some detail about how to create a useful bug report. This describes what kind of information is useful and why it is useful. - `Bug Writing Guidelines `_ + `Bug Report Writing Guidelines `_ Information about writing a good bug report. Some of this is specific to the Mozilla project, but describes general good practices. diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 8e8a92003c5a3e..33b0c06a9ebc2e 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -48,7 +48,7 @@ Allocating Objects on the Heap improving the memory management efficiency. -.. c:function:: void PyObject_Del(PyObject *op) +.. c:function:: void PyObject_Del(void *op) Releases memory allocated to an object using :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`. This is normally called from the diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 72d965053cfda3..fdb8bd19d218d2 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -355,8 +355,10 @@ If :c:member:`~Py_buffer.strides` is *NULL*, the array is interpreted as a standard n-dimensional C-array. Otherwise, the consumer must access an n-dimensional array as follows: - ``ptr = (char *)buf + indices[0] * strides[0] + ... + indices[n-1] * strides[n-1]`` - ``item = *((typeof(item) *)ptr);`` +.. code-block:: c + + ptr = (char *)buf + indices[0] * strides[0] + ... + indices[n-1] * strides[n-1]; + item = *((typeof(item) *)ptr); As noted above, :c:member:`~Py_buffer.buf` can point to any location within @@ -473,6 +475,12 @@ Buffer-related functions (*order* is ``'A'``). Return ``0`` otherwise. This function always succeeds. +.. c:function:: void* PyBuffer_GetPointer(Py_buffer *view, Py_ssize_t *indices) + + Get the memory area pointed to by the *indices* inside the given *view*. + *indices* must point to an array of ``view->ndim`` indices. + + .. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssize_t len, char order) Copy *len* bytes from *src* to its contiguous representation in *buf*. diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 48428109e6f2fe..3c4f66923da5a9 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -33,19 +33,20 @@ bound into a function. Return the number of free variables in *co*. -.. c:function:: PyCodeObject* PyCode_New(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) +.. c:function:: PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) - Return a new code object. If you need a dummy code object to - create a frame, use :c:func:`PyCode_NewEmpty` instead. Calling - :c:func:`PyCode_New` directly can bind you to a precise Python - version since the definition of the bytecode changes often. + Return a new code object. If you need a dummy code object to create a frame, + use :c:func:`PyCode_NewEmpty` instead. Calling :c:func:`PyCode_New` directly + can bind you to a precise Python version since the definition of the bytecode + changes often. - .. versionchanged:: 3.8 - An extra parameter is required (*posonlyargcount*) to support :PEP:`570`. - The first parameter (*argcount*) now represents the total number of positional arguments, - including positional-only. + .. audit-event:: code.__new__ code,filename,name,argcount,posonlyargcount,kwonlyargcount,nlocals,stacksize,flags c.PyCode_New - .. audit-event:: code.__new__ "code filename name argcount posonlyargcount kwonlyargcount nlocals stacksize flags" +.. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) + + Similar to :c:func:`PyCode_New`, but with an extra "posonlyargcount" for positonal-only arguments. + + .. versionadded:: 3.8 .. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index d3f6daa8347e41..25bb657dca0e16 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -320,7 +320,7 @@ an error value). :mod:`warnings` module and the :option:`-W` option in the command line documentation. There is no C API for warning control. -.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *msg, PyObject *name, PyObject *path) +.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *exception, PyObject *msg, PyObject *name, PyObject *path) Much like :c:func:`PyErr_SetImportError` but this function allows for specifying a subclass of :exc:`ImportError` to raise. @@ -809,7 +809,6 @@ the variables: single: PyExc_SystemError single: PyExc_SystemExit single: PyExc_TabError - single: PyExc_TargetScopeError single: PyExc_TimeoutError single: PyExc_TypeError single: PyExc_UnboundLocalError @@ -911,8 +910,6 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TabError` | :exc:`TabError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TargetScopeError` | :exc:`TargetScopeError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TypeError` | :exc:`TypeError` | | diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index a0ac4d21d13936..e0558f426dce36 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -7,6 +7,8 @@ Initialization, Finalization, and Threads ***************************************** +See also :ref:`Python Initialization Configuration `. + .. _pre-init-safe: Before Python Initialization @@ -1141,10 +1143,18 @@ Sub-interpreter support While in most uses, you will only embed a single Python interpreter, there are cases where you need to create several independent interpreters in the -same process and perhaps even in the same thread. Sub-interpreters allow -you to do that. You can switch between sub-interpreters using the -:c:func:`PyThreadState_Swap` function. You can create and destroy them -using the following functions: +same process and perhaps even in the same thread. Sub-interpreters allow +you to do that. + +The "main" interpreter is the first one created when the runtime initializes. +It is usually the only Python interpreter in a process. Unlike sub-interpreters, +the main interpreter has unique process-global responsibilities like signal +handling. It is also responsible for execution during runtime initialization and +is usually the active interpreter during runtime finalization. The +:c:func:`PyInterpreterState_Main` funtion returns a pointer to its state. + +You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap` +function. You can create and destroy them using the following functions: .. c:function:: PyThreadState* Py_NewInterpreter() diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 0d94e6b8f27b57..c198df3fb1d083 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -25,6 +25,7 @@ Functions: * :c:func:`PyConfig_SetBytesArgv` * :c:func:`PyConfig_SetBytesString` * :c:func:`PyConfig_SetString` +* :c:func:`PyConfig_SetWideStringList` * :c:func:`PyPreConfig_InitIsolatedConfig` * :c:func:`PyPreConfig_InitPythonConfig` * :c:func:`PyStatus_Error` @@ -47,6 +48,8 @@ The preconfiguration (``PyPreConfig`` type) is stored in ``_PyRuntime.preconfig`` and the configuration (``PyConfig`` type) is stored in ``PyInterpreterState.config``. +See also :ref:`Initialization, Finalization, and Threads `. + .. seealso:: :pep:`587` "Python Initialization Configuration". @@ -71,8 +74,12 @@ PyWideStringList .. c:function:: PyStatus PyWideStringList_Insert(PyWideStringList *list, Py_ssize_t index, const wchar_t *item) - Insert *item* into *list* at *index*. If *index* is greater than *list* - length, just append *item* to *list*. + Insert *item* into *list* at *index*. + + If *index* is greater than or equal to *list* length, append *item* to + *list*. + + *index* must be greater than or equal to 0. Python must be preinitialized to call this function. @@ -368,6 +375,12 @@ PyConfig Preinitialize Python if needed. + .. c:function:: PyStatus PyConfig_SetWideStringList(PyConfig *config, PyWideStringList *list, Py_ssize_t length, wchar_t **items) + + Set the list of wide strings *list* to *length* and *items*. + + Preinitialize Python if needed. + .. c:function:: PyStatus PyConfig_Read(PyConfig *config) Read all Python configuration. @@ -863,7 +876,7 @@ Path Configuration If at least one "output field" is not set, Python computes the path configuration to fill unset fields. If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, -:c:member:`~PyConfig.module_search_paths` is overriden and +:c:member:`~PyConfig.module_search_paths` is overridden and :c:member:`~PyConfig.module_search_paths_set` is set to 1. It is possible to completely ignore the function computing the default diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 279783a243a108..dc9026605be358 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -59,9 +59,9 @@ List Objects .. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) Return the object at position *index* in the list pointed to by *list*. The - position must be positive, indexing from the end of the list is not - supported. If *index* is out of bounds, return *NULL* and set an - :exc:`IndexError` exception. + position must be non-negative; indexing from the end of the list is not + supported. If *index* is out of bounds (<0 or >=len(list)), + return *NULL* and set an :exc:`IndexError` exception. .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 6be29f9cd32eaa..fdaefafe21ba09 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -288,7 +288,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. If the value of *obj* is out of range for an :c:type:`unsigned long`, return the reduction of that value modulo ``ULONG_MAX + 1``. - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``(unsigned long)-1`` on error. Use :c:func:`PyErr_Occurred` to + disambiguate. .. versionchanged:: 3.8 Use :meth:`__index__` if available. @@ -307,7 +308,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. If the value of *obj* is out of range for an :c:type:`unsigned long long`, return the reduction of that value modulo ``PY_ULLONG_MAX + 1``. - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``(unsigned long long)-1`` on error. Use :c:func:`PyErr_Occurred` + to disambiguate. .. versionchanged:: 3.8 Use :meth:`__index__` if available. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index ab49e48782d71a..d3c8b30d360ec2 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -67,7 +67,7 @@ example:: In this example, the memory request for the I/O buffer is handled by the C library allocator. The Python memory manager is involved only in the allocation -of the string object returned as a result. +of the bytes object returned as a result. In most situations, however, it is recommended to allocate memory from the Python heap specifically because the latter is under control of the Python @@ -472,7 +472,7 @@ Customize Memory Allocators if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and :c:data:`PYMEM_DOMAIN_MEM` domains are called. - .. versionchanged:: 3.8.0 + .. versionchanged:: 3.8 Byte patterns ``0xCB`` (``CLEANBYTE``), ``0xDB`` (``DEADBYTE``) and ``0xFB`` (``FORBIDDENBYTE``) have been replaced with ``0xCD``, ``0xDD`` and ``0xFD`` to use the same values than Windows CRT debug ``malloc()`` diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index ce0d05942f4e79..187a025a67cb6d 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -261,7 +261,8 @@ Object Protocol *args* must not be *NULL*, use an empty tuple if no arguments are needed. If no named arguments are needed, *kwargs* can be *NULL*. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This is the equivalent of the Python expression: ``callable(*args, **kwargs)``. @@ -272,7 +273,8 @@ Object Protocol Call a callable Python object *callable*, with arguments given by the tuple *args*. If no arguments are needed, then *args* can be *NULL*. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This is the equivalent of the Python expression: ``callable(*args)``. @@ -283,7 +285,8 @@ Object Protocol The C arguments are described using a :c:func:`Py_BuildValue` style format string. The format can be *NULL*, indicating that no arguments are provided. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This is the equivalent of the Python expression: ``callable(*args)``. @@ -302,7 +305,8 @@ Object Protocol The format can be *NULL*, indicating that no arguments are provided. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This is the equivalent of the Python expression: ``obj.name(arg1, arg2, ...)``. @@ -320,7 +324,8 @@ Object Protocol :c:type:`PyObject\*` arguments. The arguments are provided as a variable number of parameters followed by *NULL*. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This is the equivalent of the Python expression: ``callable(arg1, arg2, ...)``. @@ -331,7 +336,9 @@ Object Protocol Calls a method of the Python object *obj*, where the name of the method is given as a Python string object in *name*. It is called with a variable number of :c:type:`PyObject\*` arguments. The arguments are provided as a variable number - of parameters followed by *NULL*. Returns the result of the call on success, or + of parameters followed by *NULL*. + + Return the result of the call on success, or raise an exception and return *NULL* on failure. @@ -355,7 +362,8 @@ Object Protocol *kwnames* must contain only objects of type ``str`` (not a subclass), and all keys must be unique. - Return the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This uses the vectorcall protocol if the callable supports it; otherwise, the arguments are converted to use diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 6b07c87d62eded..ec9cfc6f0e625c 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -68,6 +68,6 @@ simply exported function versions of :c:func:`Py_XINCREF` and :c:func:`Py_XDECREF`, respectively. The following functions or macros are only for use within the interpreter core: -:c:func:`_Py_Dealloc`, :c:func:`_Py_ForgetReference`, :c:func:`_Py_NewReference`, +:c:func:`_Py_ForgetReference`, :c:func:`_Py_NewReference`, as well as the global variable :c:data:`_Py_RefTotal`. diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 5e0cfd0264f931..5184ad511cd9b1 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -114,10 +114,20 @@ the definition of all other Python objects. .. c:type:: PyCFunctionWithKeywords - Type of the functions used to implement Python callables in C that take - keyword arguments: they take three :c:type:`PyObject\*` parameters and return - one such value. See :c:type:`PyCFunction` above for the meaning of the return - value. + Type of the functions used to implement Python callables in C + with signature :const:`METH_VARARGS | METH_KEYWORDS`. + + +.. c:type:: _PyCFunctionFast + + Type of the functions used to implement Python callables in C + with signature :const:`METH_FASTCALL`. + + +.. c:type:: _PyCFunctionFastWithKeywords + + Type of the functions used to implement Python callables in C + with signature :const:`METH_FASTCALL | METH_KEYWORDS`. .. c:type:: PyMethodDef @@ -149,10 +159,11 @@ specific C type of the *self* object. The :attr:`ml_flags` field is a bitfield which can include the following flags. The individual flags indicate either a calling convention or a binding -convention. Of the calling convention flags, only :const:`METH_VARARGS` and -:const:`METH_KEYWORDS` can be combined. Any of the calling convention flags -can be combined with a binding flag. +convention. +There are four basic calling conventions for positional arguments +and two of them can be combined with :const:`METH_KEYWORDS` to support +also keyword arguments. So there are a total of 6 calling conventions: .. data:: METH_VARARGS @@ -164,13 +175,41 @@ can be combined with a binding flag. using :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_UnpackTuple`. -.. data:: METH_KEYWORDS +.. data:: METH_VARARGS | METH_KEYWORDS Methods with these flags must be of type :c:type:`PyCFunctionWithKeywords`. - The function expects three parameters: *self*, *args*, and a dictionary of - all the keyword arguments. The flag must be combined with - :const:`METH_VARARGS`, and the parameters are typically processed using - :c:func:`PyArg_ParseTupleAndKeywords`. + The function expects three parameters: *self*, *args*, *kwargs* where + *kwargs* is a dictionary of all the keyword arguments or possibly *NULL* + if there are no keyword arguments. The parameters are typically processed + using :c:func:`PyArg_ParseTupleAndKeywords`. + + +.. data:: METH_FASTCALL + + Fast calling convention supporting only positional arguments. + The methods have the type :c:type:`_PyCFunctionFast`. + The first parameter is *self*, the second parameter is a C array + of :c:type:`PyObject\*` values indicating the arguments and the third + parameter is the number of arguments (the length of the array). + + This is not part of the :ref:`limited API `. + + .. versionadded:: 3.7 + + +.. data:: METH_FASTCALL | METH_KEYWORDS + + Extension of :const:`METH_FASTCALL` supporting also keyword arguments, + with methods of type :c:type:`_PyCFunctionFastWithKeywords`. + Keyword arguments are passed the same way as in the vectorcall protocol: + there is an additional fourth :c:type:`PyObject\*` parameter + which is a tuple representing the names of the keyword arguments + or possibly *NULL* if there are no keywords. The values of the keyword + arguments are stored in the *args* array, after the positional arguments. + + This is not part of the :ref:`limited API `. + + .. versionadded:: 3.7 .. data:: METH_NOARGS diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 7d870a8d4e4a71..838a97cacfdf10 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -291,8 +291,6 @@ accessible to C code. They all work with the current interpreter thread's .. c:function:: int PySys_Audit(const char *event, const char *format, ...) - .. index:: single: audit events - Raises an auditing event with any active hooks. Returns zero for success and non-zero with an exception set on failure. @@ -311,8 +309,6 @@ accessible to C code. They all work with the current interpreter thread's .. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) - .. index:: single: audit events - Adds to the collection of active auditing hooks. Returns zero for success and non-zero on failure. If the runtime has been initialized, also sets an error on failure. Hooks added through this API are called for all diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 83fcc5abed7079..638fb0c32d788b 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -285,7 +285,7 @@ sub-slots +---------------------------------------------------------+-----------------------------------+--------------+ | :c:member:`~PyNumberMethods.nb_inplace_true_divide` | :c:type:`binaryfunc` | __truediv__ | +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_index` | :c:type:`binaryfunc` | __index__ | + | :c:member:`~PyNumberMethods.nb_index` | :c:type:`unaryfunc` | __index__ | +---------------------------------------------------------+-----------------------------------+--------------+ | :c:member:`~PyNumberMethods.nb_matrix_multiply` | :c:type:`binaryfunc` | __matmul__ | | | | __rmatmul__ | diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 3fe0ae47aac35a..67dc11dfa9a537 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -335,12 +335,12 @@ the same library that the Python runtime is using. .. c:function:: PyObject* PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) - This is the main, unvarnished function of Python interpretation. It is - literally 2000 lines long. The code object associated with the execution - frame *f* is executed, interpreting bytecode and executing calls as needed. - The additional *throwflag* parameter can mostly be ignored - if true, then - it causes an exception to immediately be thrown; this is used for the - :meth:`~generator.throw` methods of generator objects. + This is the main, unvarnished function of Python interpretation. The code + object associated with the execution frame *f* is executed, interpreting + bytecode and executing calls as needed. The additional *throwflag* + parameter can mostly be ignored - if true, then it causes an exception + to immediately be thrown; this is used for the :meth:`~generator.throw` + methods of generator objects. .. versionchanged:: 3.4 This function now includes a debug assertion to help ensure that it @@ -388,11 +388,22 @@ the same library that the Python runtime is using. Whenever ``PyCompilerFlags *flags`` is *NULL*, :attr:`cf_flags` is treated as equal to ``0``, and any modification due to ``from __future__ import`` is - discarded. :: + discarded. - struct PyCompilerFlags { - int cf_flags; - } + .. c:member:: int cf_flags + + Compiler flags. + + .. c:member:: int cf_feature_version + + *cf_feature_version* is the minor Python version. It should be + initialized to ``PY_MINOR_VERSION``. + + The field is ignored by default, it is used if and only if + ``PyCF_ONLY_AST`` flag is set in *cf_flags*. + + .. versionchanged:: 3.8 + Added *cf_feature_version* field. .. c:var:: int CO_FUTURE_DIVISION diff --git a/Doc/conf.py b/Doc/conf.py index e85ea5b2d2ff49..abaa760c98c1aa 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -48,8 +48,10 @@ needs_sphinx = '1.8' # Ignore any .rst files in the venv/ directory. -venvdir = os.getenv('VENVDIR', 'venv') -exclude_patterns = [venvdir+'/*', 'README.rst'] +exclude_patterns = ['venv/*', 'README.rst'] +venvdir = os.getenv('VENVDIR') +if venvdir is not None: + exclude_patterns.append(venvdir + '/*') # Disable Docutils smartquotes for several translations smartquotes_excludes = { diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index aca57a1dae9d8c..cfed1bd503118e 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -234,9 +234,26 @@ PyCode_Check:PyObject*:co:0: PyCode_GetNumFree:int::: PyCode_GetNumFree:PyCodeObject*:co:0: +PyCode_NewWithPosOnlyArgs:PyCodeObject*::+1: +PyCode_NewWithPosOnlyArgs:int:argcount:: +PyCode_NewWithPosOnlyArgs:int:posonlyargcount:: +PyCode_NewWithPosOnlyArgs:int:kwonlyargcount:: +PyCode_NewWithPosOnlyArgs:int:nlocals:: +PyCode_NewWithPosOnlyArgs:int:stacksize:: +PyCode_NewWithPosOnlyArgs:int:flags:: +PyCode_NewWithPosOnlyArgs:PyObject*:code:0: +PyCode_NewWithPosOnlyArgs:PyObject*:consts:0: +PyCode_NewWithPosOnlyArgs:PyObject*:names:0: +PyCode_NewWithPosOnlyArgs:PyObject*:varnames:0: +PyCode_NewWithPosOnlyArgs:PyObject*:freevars:0: +PyCode_NewWithPosOnlyArgs:PyObject*:cellvars:0: +PyCode_NewWithPosOnlyArgs:PyObject*:filename:0: +PyCode_NewWithPosOnlyArgs:PyObject*:name:0: +PyCode_NewWithPosOnlyArgs:int:firstlineno:: +PyCode_NewWithPosOnlyArgs:PyObject*:lnotab:0: + PyCode_New:PyCodeObject*::+1: PyCode_New:int:argcount:: -PyCode_New:int:posonlyargcount:: PyCode_New:int:kwonlyargcount:: PyCode_New:int:nlocals:: PyCode_New:int:stacksize:: @@ -2445,7 +2462,7 @@ PyUnicode_FromWideChar:Py_ssize_t:size:: PyUnicode_AsWideChar:Py_ssize_t::: PyUnicode_AsWideChar:PyObject*:*unicode:0: PyUnicode_AsWideChar:wchar_t*:w:: -PyUnicode_AsWideChar:Pyssize_t:size:: +PyUnicode_AsWideChar:Py_ssize_t:size:: PyUnicode_AsWideCharString:wchar_t*::: PyUnicode_AsWideCharString:PyObject*:unicode:0: diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index 2601d30f63eb4c..937f19f57be95a 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -1863,6 +1863,9 @@ Subclasses of :class:`Command` must define the following methods. .. module:: distutils.command.bdist_wininst :synopsis: Build a Windows installer +.. deprecated:: 3.8 + Use bdist_wheel (wheel packages) instead. + .. % todo diff --git a/Doc/distutils/builtdist.rst b/Doc/distutils/builtdist.rst index f44d0d039f45f4..b814f2e9508c9c 100644 --- a/Doc/distutils/builtdist.rst +++ b/Doc/distutils/builtdist.rst @@ -146,6 +146,9 @@ generated by each, are: | :command:`bdist_msi` | msi | +--------------------------+-------------------------------------+ +.. note:: + bdist_wininst is deprecated since Python 3.8. + The following sections give details on the individual :command:`bdist_\*` commands. @@ -298,6 +301,9 @@ file winds up deep in the "build tree," in a temporary directory created by Creating Windows Installers =========================== +.. warning:: + bdist_wininst is deprecated since Python 3.8. + Executable installers are the natural format for binary distributions on Windows. They display a nice graphical user interface, display some information about the module distribution to be installed taken from the metadata in the @@ -315,8 +321,8 @@ or the :command:`bdist` command with the :option:`!--formats` option:: If you have a pure module distribution (only containing pure Python modules and packages), the resulting installer will be version independent and have a name -like :file:`foo-1.0.win32.exe`. These installers can even be created on Unix -platforms or Mac OS X. +like :file:`foo-1.0.win32.exe`. Note that creating ``wininst`` binary +distributions in only supported on Windows systems. If you have a non-pure distribution, the extensions can only be created on a Windows platform, and will be Python version dependent. The installer filename @@ -459,3 +465,6 @@ Starting with Python 2.6, bdist_wininst supports a :option:`!--user-access-contr option. The default is 'none' (meaning no UAC handling is done), and other valid values are 'auto' (meaning prompt for UAC elevation if Python was installed for all users) and 'force' (meaning always prompt for elevation). + +.. note:: + bdist_wininst is deprecated since Python 3.8. diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index a00c6a053ef120..2ff7236df1dc0e 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -659,7 +659,7 @@ How can my code discover the name of an object? ----------------------------------------------- Generally speaking, it can't, because objects don't really have names. -Essentially, assignment always binds a name to a value; The same is true of +Essentially, assignment always binds a name to a value; the same is true of ``def`` and ``class`` statements, but in that case the value is a callable. Consider the following code:: @@ -800,10 +800,6 @@ an error:: File "", line 1, in TypeError: pow() takes no keyword arguments -Note that as of this writing this is only documentational and no valid syntax -in Python, although there is :pep:`570`, which proposes a syntax for -position-only parameters in Python. - Numbers and strings =================== diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index b29e590b20cba8..9856e6caec0928 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -117,7 +117,7 @@ The important points to remember are: * non-data descriptors may be overridden by instance dictionaries. The object returned by ``super()`` also has a custom :meth:`__getattribute__` -method for invoking descriptors. The call ``super(B, obj).m()`` searches +method for invoking descriptors. The attribute lookup ``super(B, obj).m`` searches ``obj.__class__.__mro__`` for the base class ``A`` immediately following ``B`` and then returns ``A.__dict__['m'].__get__(obj, B)``. If not a descriptor, ``m`` is returned unchanged. If not in the dictionary, ``m`` reverts to a diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 71f9fc920fdfc9..9f52780af4fc2d 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -948,6 +948,41 @@ This variant shows how you can e.g. apply configuration for particular loggers machinery in the main process (even though the logging events are generated in the worker processes) to direct the messages to the appropriate destinations. +Using concurrent.futures.ProcessPoolExecutor +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you want to use :class:`concurrent.futures.ProcessPoolExecutor` to start +your worker processes, you need to create the queue slightly differently. +Instead of + +.. code-block:: python + + queue = multiprocessing.Queue(-1) + +you should use + +.. code-block:: python + + queue = multiprocessing.Manager().Queue(-1) # also works with the examples above + +and you can then replace the worker creation from this:: + + workers = [] + for i in range(10): + worker = multiprocessing.Process(target=worker_process, + args=(queue, worker_configurer)) + workers.append(worker) + worker.start() + for w in workers: + w.join() + +to this (remembering to first import :mod:`concurrent.futures`):: + + with concurrent.futures.ProcessPoolExecutor(max_workers=10) as executor: + for i in range(10): + executor.submit(worker_process, queue, worker_configurer) + + Using file rotation ------------------- @@ -2266,9 +2301,9 @@ The script just arranges to decorate ``foo`` with a decorator which will do the conditional logging that's required. The decorator takes a logger as a parameter and attaches a memory handler for the duration of the call to the decorated function. The decorator can be additionally parameterised using a target handler, -a level at which flushing should occur, and a capacity for the buffer. These -default to a :class:`~logging.StreamHandler` which writes to ``sys.stderr``, -``logging.ERROR`` and ``100`` respectively. +a level at which flushing should occur, and a capacity for the buffer (number of +records buffered). These default to a :class:`~logging.StreamHandler` which +writes to ``sys.stderr``, ``logging.ERROR`` and ``100`` respectively. Here's the script:: @@ -2709,3 +2744,238 @@ And if we want less: In this case, the commands don't print anything to the console, since nothing at ``WARNING`` level or above is logged by them. + +.. _qt-gui: + +A Qt GUI for logging +-------------------- + +A question that comes up from time to time is about how to log to a GUI +application. The `Qt `_ framework is a popular +cross-platform UI framework with Python bindings using `PySide2 +`_ or `PyQt5 +`_ libraries. + +The following example shows how to log to a Qt GUI. This introduces a simple +``QtHandler`` class which takes a callable, which should be a slot in the main +thread that does GUI updates. A worker thread is also created to show how you +can log to the GUI from both the UI itself (via a button for manual logging) +as well as a worker thread doing work in the background (here, just logging +messages at random levels with random short delays in between). + +The worker thread is implemented using Qt's ``QThread`` class rather than the +:mod:`threading` module, as there are circumstances where one has to use +``QThread``, which offers better integration with other ``Qt`` components. + +The code should work with recent releases of either ``PySide2`` or ``PyQt5``. +You should be able to adapt the approach to earlier versions of Qt. Please +refer to the comments in the code snippet for more detailed information. + +.. code-block:: python3 + + import datetime + import logging + import random + import sys + import time + + # Deal with minor differences between PySide2 and PyQt5 + try: + from PySide2 import QtCore, QtGui, QtWidgets + Signal = QtCore.Signal + Slot = QtCore.Slot + except ImportError: + from PyQt5 import QtCore, QtGui, QtWidgets + Signal = QtCore.pyqtSignal + Slot = QtCore.pyqtSlot + + + logger = logging.getLogger(__name__) + + + # + # Signals need to be contained in a QObject or subclass in order to be correctly + # initialized. + # + class Signaller(QtCore.QObject): + signal = Signal(str, logging.LogRecord) + + # + # Output to a Qt GUI is only supposed to happen on the main thread. So, this + # handler is designed to take a slot function which is set up to run in the main + # thread. In this example, the function takes a string argument which is a + # formatted log message, and the log record which generated it. The formatted + # string is just a convenience - you could format a string for output any way + # you like in the slot function itself. + # + # You specify the slot function to do whatever GUI updates you want. The handler + # doesn't know or care about specific UI elements. + # + class QtHandler(logging.Handler): + def __init__(self, slotfunc, *args, **kwargs): + super(QtHandler, self).__init__(*args, **kwargs) + self.signaller = Signaller() + self.signaller.signal.connect(slotfunc) + + def emit(self, record): + s = self.format(record) + self.signaller.signal.emit(s, record) + + # + # This example uses QThreads, which means that the threads at the Python level + # are named something like "Dummy-1". The function below gets the Qt name of the + # current thread. + # + def ctname(): + return QtCore.QThread.currentThread().objectName() + + + # + # Used to generate random levels for logging. + # + LEVELS = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, + logging.CRITICAL) + + # + # This worker class represents work that is done in a thread separate to the + # main thread. The way the thread is kicked off to do work is via a button press + # that connects to a slot in the worker. + # + # Because the default threadName value in the LogRecord isn't much use, we add + # a qThreadName which contains the QThread name as computed above, and pass that + # value in an "extra" dictionary which is used to update the LogRecord with the + # QThread name. + # + # This example worker just outputs messages sequentially, interspersed with + # random delays of the order of a few seconds. + # + class Worker(QtCore.QObject): + @Slot() + def start(self): + extra = {'qThreadName': ctname() } + logger.debug('Started work', extra=extra) + i = 1 + # Let the thread run until interrupted. This allows reasonably clean + # thread termination. + while not QtCore.QThread.currentThread().isInterruptionRequested(): + delay = 0.5 + random.random() * 2 + time.sleep(delay) + level = random.choice(LEVELS) + logger.log(level, 'Message after delay of %3.1f: %d', delay, i, extra=extra) + i += 1 + + # + # Implement a simple UI for this cookbook example. This contains: + # + # * A read-only text edit window which holds formatted log messages + # * A button to start work and log stuff in a separate thread + # * A button to log something from the main thread + # * A button to clear the log window + # + class Window(QtWidgets.QWidget): + + COLORS = { + logging.DEBUG: 'black', + logging.INFO: 'blue', + logging.WARNING: 'orange', + logging.ERROR: 'red', + logging.CRITICAL: 'purple', + } + + def __init__(self, app): + super(Window, self).__init__() + self.app = app + self.textedit = te = QtWidgets.QPlainTextEdit(self) + # Set whatever the default monospace font is for the platform + f = QtGui.QFont('nosuchfont') + f.setStyleHint(f.Monospace) + te.setFont(f) + te.setReadOnly(True) + PB = QtWidgets.QPushButton + self.work_button = PB('Start background work', self) + self.log_button = PB('Log a message at a random level', self) + self.clear_button = PB('Clear log window', self) + self.handler = h = QtHandler(self.update_status) + # Remember to use qThreadName rather than threadName in the format string. + fs = '%(asctime)s %(qThreadName)-12s %(levelname)-8s %(message)s' + formatter = logging.Formatter(fs) + h.setFormatter(formatter) + logger.addHandler(h) + # Set up to terminate the QThread when we exit + app.aboutToQuit.connect(self.force_quit) + + # Lay out all the widgets + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(te) + layout.addWidget(self.work_button) + layout.addWidget(self.log_button) + layout.addWidget(self.clear_button) + self.setFixedSize(900, 400) + + # Connect the non-worker slots and signals + self.log_button.clicked.connect(self.manual_update) + self.clear_button.clicked.connect(self.clear_display) + + # Start a new worker thread and connect the slots for the worker + self.start_thread() + self.work_button.clicked.connect(self.worker.start) + # Once started, the button should be disabled + self.work_button.clicked.connect(lambda : self.work_button.setEnabled(False)) + + def start_thread(self): + self.worker = Worker() + self.worker_thread = QtCore.QThread() + self.worker.setObjectName('Worker') + self.worker_thread.setObjectName('WorkerThread') # for qThreadName + self.worker.moveToThread(self.worker_thread) + # This will start an event loop in the worker thread + self.worker_thread.start() + + def kill_thread(self): + # Just tell the worker to stop, then tell it to quit and wait for that + # to happen + self.worker_thread.requestInterruption() + if self.worker_thread.isRunning(): + self.worker_thread.quit() + self.worker_thread.wait() + else: + print('worker has already exited.') + + def force_quit(self): + # For use when the window is closed + if self.worker_thread.isRunning(): + self.kill_thread() + + # The functions below update the UI and run in the main thread because + # that's where the slots are set up + + @Slot(str) + def update_status(self, status, record): + color = self.COLORS.get(record.levelno, 'black') + s = '
%s
' % (color, status) + self.textedit.appendHtml(s) + + @Slot() + def manual_update(self): + # This function uses the formatted message passed in, but also uses + # information from the record to format the message in an appropriate + # color according to its severity (level). + level = random.choice(LEVELS) + extra = {'qThreadName': ctname() } + logger.log(level, 'Manually logged!', extra=extra) + + @Slot() + def clear_display(self): + self.textedit.clear() + + + def main(): + QtCore.QThread.currentThread().setObjectName('MainThread') + logging.getLogger().setLevel(logging.DEBUG) + app = QtWidgets.QApplication(sys.argv) + example = Window(app) + example.show() + sys.exit(app.exec_()) + + if __name__=='__main__': + main() diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 24c3235e4add94..51bd64bfc232ca 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -57,14 +57,14 @@ their corresponding code points: ... 007B '{'; LEFT CURLY BRACKET ... - 2167 'Ⅶ': ROMAN NUMERAL EIGHT - 2168 'Ⅸ': ROMAN NUMERAL NINE + 2167 'Ⅷ'; ROMAN NUMERAL EIGHT + 2168 'Ⅸ'; ROMAN NUMERAL NINE ... - 265E '♞': BLACK CHESS KNIGHT - 265F '♟': BLACK CHESS PAWN + 265E '♞'; BLACK CHESS KNIGHT + 265F '♟'; BLACK CHESS PAWN ... - 1F600 '😀': GRINNING FACE - 1F609 '😉': WINKING FACE + 1F600 '😀'; GRINNING FACE + 1F609 '😉'; WINKING FACE ... Strictly, these definitions imply that it's meaningless to say 'this is diff --git a/Doc/includes/email-mime.py b/Doc/includes/email-mime.py index c610242f11f843..6af2be0b08a48d 100644 --- a/Doc/includes/email-mime.py +++ b/Doc/includes/email-mime.py @@ -14,7 +14,7 @@ # family = the list of all recipients' email addresses msg['From'] = me msg['To'] = ', '.join(family) -msg.preamble = 'Our family reunion' +msg.preamble = 'You will not see this in a MIME-aware mail reader.\n' # Open the files in binary mode. Use imghdr to figure out the # MIME subtype for each specific image. diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 26568dcbf8f532..bd653ab32bb9c4 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -106,7 +106,7 @@ This module defines the following constants and functions: Its value may be used to uniquely identify this particular thread system-wide (until the thread terminates, after which the value may be recycled by the OS). - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD. + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. .. versionadded:: 3.8 diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index b77a38ccd48577..368b1cfebf0568 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -182,6 +182,10 @@ ArgumentParser objects .. versionchanged:: 3.5 *allow_abbrev* parameter was added. + .. versionchanged:: 3.8 + In previous versions, *allow_abbrev* also disabled grouping of short + flags such as ``-vv`` to mean ``-v -v``. + The following sections describe how each of these are used. @@ -1098,9 +1102,8 @@ container should match the type_ specified:: usage: doors.py [-h] {1,2,3} doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3) -Any object that supports the ``in`` operator can be passed as the *choices* -value, so :class:`dict` objects, :class:`set` objects, custom containers, -etc. are all supported. +Any container can be passed as the *choices* value, so :class:`list` objects, +:class:`set` objects, and custom containers are all supported. required diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 1f95dd61b9fcd7..2ae2a071262a17 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -36,9 +36,9 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'L'`` | unsigned long | int | 4 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'q'`` | signed long long | int | 8 | \(2) | +| ``'q'`` | signed long long | int | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'Q'`` | unsigned long long | int | 8 | \(2) | +| ``'Q'`` | unsigned long long | int | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'f'`` | float | float | 4 | | +-----------+--------------------+-------------------+-----------------------+-------+ @@ -57,13 +57,6 @@ Notes: .. deprecated-removed:: 3.3 4.0 -(2) - The ``'q'`` and ``'Q'`` type codes are available only if - the platform C compiler used to build Python supports C :c:type:`long long`, - or, on Windows, :c:type:`__int64`. - - .. versionadded:: 3.3 - The actual representation of values is determined by the machine architecture (strictly speaking, by the C implementation). The actual size can be accessed through the :attr:`itemsize` attribute. @@ -83,7 +76,7 @@ The module defines the following type: to add initial items to the array. Otherwise, the iterable initializer is passed to the :meth:`extend` method. - .. audit-event:: array.__new__ "typecode initializer" + .. audit-event:: array.__new__ typecode,initializer array.array .. data:: typecodes diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 1884bea80e8047..92bf8912eb53dd 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -126,7 +126,7 @@ The abstract grammar is currently defined as follows: Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=-1) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None) Parse the source into an AST node. Equivalent to ``compile(source, filename, mode, ast.PyCF_ONLY_AST)``. @@ -145,11 +145,12 @@ and classes for traversing abstract syntax trees: modified to correspond to :pep:`484` "signature type comments", e.g. ``(str, int) -> List[str]``. - Also, setting ``feature_version`` to the minor version of an - earlier Python 3 version will attempt to parse using that version's - grammar. For example, setting ``feature_version=4`` will allow - the use of ``async`` and ``await`` as variable names. The lowest - supported value is 4; the highest is ``sys.version_info[1]``. + Also, setting ``feature_version`` to a tuple ``(major, minor)`` + will attempt to parse using that Python version's grammar. + Currently ``major`` must equal to ``3``. For example, setting + ``feature_version=(3, 4)`` will allow the use of ``async`` and + ``await`` as variable names. The lowest supported version is + ``(3, 4)``; the highest is ``sys.version_info[0:2]``. .. warning:: It is possible to crash the Python interpreter with a @@ -274,6 +275,13 @@ and classes for traversing abstract syntax trees: during traversal. For this a special visitor exists (:class:`NodeTransformer`) that allows modifications. + .. deprecated:: 3.8 + + Methods :meth:`visit_Num`, :meth:`visit_Str`, :meth:`visit_Bytes`, + :meth:`visit_NameConstant` and :meth:`visit_Ellipsis` are deprecated + now and will not be called in future Python versions. Add the + :meth:`visit_Constant` method to handle all constant nodes. + .. class:: NodeTransformer() @@ -314,11 +322,12 @@ and classes for traversing abstract syntax trees: .. function:: dump(node, annotate_fields=True, include_attributes=False) Return a formatted dump of the tree in *node*. This is mainly useful for - debugging purposes. The returned string will show the names and the values - for fields. This makes the code impossible to evaluate, so if evaluation is - wanted *annotate_fields* must be set to ``False``. Attributes such as line + debugging purposes. If *annotate_fields* is true (by default), + the returned string will show the names and the values for fields. + If *annotate_fields* is false, the result string will be more compact by + omitting unambiguous field names. Attributes such as line numbers and column offsets are not dumped by default. If this is wanted, - *include_attributes* can be set to ``True``. + *include_attributes* can be set to true. .. seealso:: diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index b7288036192979..101e7817a95e98 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -119,7 +119,7 @@ all concurrent asyncio Tasks and IO operations would be delayed by 1 second. An executor can be used to run a task in a different thread or even in -a different process to avoid blocking block the OS thread with the +a different process to avoid blocking the OS thread with the event loop. See the :meth:`loop.run_in_executor` method for more details. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 8673f84e96382e..8f7974be66eaa6 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1454,7 +1454,7 @@ asyncio ships with two different event loop implementations: :class:`SelectorEventLoop` and :class:`ProactorEventLoop`. By default asyncio is configured to use :class:`SelectorEventLoop` -on all platforms. +on Unix and :class:`ProactorEventLoop` on Windows. .. class:: SelectorEventLoop @@ -1625,8 +1625,7 @@ Wait until a file descriptor received some data using the :meth:`loop.create_connection` method. * Another similar :ref:`example ` - using the high-level :func:`asyncio.open_connection` function - and streams. + using the high-level :func:`asyncio.connect` function and streams. .. _asyncio_example_unix_signals: diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 6e6e0137c1bdda..c069471b519be9 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -35,7 +35,9 @@ Future Functions is used for the test.) * a :class:`Task` object wrapping *obj*, if *obj* is a - coroutine (:func:`iscoroutine` is used for the test.) + coroutine (:func:`iscoroutine` is used for the test); + in this case the coroutine will be scheduled by + ``ensure_future()``. * a :class:`Task` object that would await on *obj*, if *obj* is an awaitable (:func:`inspect.isawaitable` is used for the test.) diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 6212df85dbc10a..aa8f8f13eae021 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -117,6 +117,7 @@ asyncio ships with the following built-in policies: .. availability:: Windows. +.. _asyncio-watchers: Process Watchers ================ @@ -129,10 +130,11 @@ In asyncio, child processes are created with :func:`create_subprocess_exec` and :meth:`loop.subprocess_exec` functions. -asyncio defines the :class:`AbstractChildWatcher` abstract base class, -which child watchers should implement, and has two different -implementations: :class:`SafeChildWatcher` (configured to be used -by default) and :class:`FastChildWatcher`. +asyncio defines the :class:`AbstractChildWatcher` abstract base class, which child +watchers should implement, and has four different implementations: +:class:`ThreadedChildWatcher` (configured to be used by default), +:class:`MultiLoopChildWatcher`, :class:`SafeChildWatcher`, and +:class:`FastChildWatcher`. See also the :ref:`Subprocess and Threads ` section. @@ -184,6 +186,15 @@ implementation used by the asyncio event loop: Note: loop may be ``None``. + .. method:: is_active() + + Return ``True`` if the watcher is ready to use. + + Spawning a subprocess with *inactive* current child watcher raises + :exc:`RuntimeError`. + + .. versionadded:: 3.8 + .. method:: close() Close the watcher. @@ -191,16 +202,48 @@ implementation used by the asyncio event loop: This method has to be called to ensure that underlying resources are cleaned-up. -.. class:: SafeChildWatcher +.. class:: ThreadedChildWatcher + + This implementation starts a new waiting thread for every subprocess spawn. + + It works reliably even when the asyncio event loop is run in a non-main OS thread. + + There is no noticeable overhead when handling a big number of children (*O(1)* each + time a child terminates), but stating a thread per process requires extra memory. + + This watcher is used by default. + + .. versionadded:: 3.8 - This implementation avoids disrupting other code spawning processes +.. class:: MultiLoopChildWatcher + + This implementation registers a :py:data:`SIGCHLD` signal handler on + instantiation. That can break third-party code that installs a custom handler for + `SIGCHLD`. signal). + + The watcher avoids disrupting other code spawning processes by polling every process explicitly on a :py:data:`SIGCHLD` signal. - This is a safe solution but it has a significant overhead when + There is no limitation for running subprocesses from different threads once the + watcher is installed. + + The solution is safe but it has a significant overhead when handling a big number of processes (*O(n)* each time a :py:data:`SIGCHLD` is received). - asyncio uses this safe implementation by default. + .. versionadded:: 3.8 + +.. class:: SafeChildWatcher + + This implementation uses active event loop from the main thread to handle + :py:data:`SIGCHLD` signal. If the main thread has no running event loop another + thread cannot spawn a subprocess (:exc:`RuntimeError` is raised). + + The watcher avoids disrupting other code spawning processes + by polling every process explicitly on a :py:data:`SIGCHLD` signal. + + This solution is as safe as :class:`MultiLoopChildWatcher` and has the same *O(N)* + complexity but requires a running event loop in the main thread to work. .. class:: FastChildWatcher @@ -211,6 +254,9 @@ implementation used by the asyncio event loop: There is no noticeable overhead when handling a big number of children (*O(1)* each time a child terminates). + This solution requires a running event loop in the main thread to work, as + :class:`SafeChildWatcher`. + Custom Policies =============== diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index f08738dd62bb5c..3e5a4dd8b891f5 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -810,7 +810,7 @@ data, and waits until the connection is closed:: .. seealso:: The :ref:`TCP echo client using streams ` - example uses the high-level :func:`asyncio.open_connection` function. + example uses the high-level :func:`asyncio.connect` function. .. _asyncio-udp-echo-server-protocol: @@ -977,7 +977,7 @@ Wait until a socket receives data using the The :ref:`register an open socket to wait for data using streams ` example uses high-level streams - created by the :func:`open_connection` function in a coroutine. + created by the :func:`asyncio.connect` function in a coroutine. .. _asyncio_example_subprocess_proto: diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 28ca5d5f339692..dfe520de56bf04 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -18,17 +18,12 @@ streams:: import asyncio async def tcp_echo_client(message): - reader, writer = await asyncio.open_connection( - '127.0.0.1', 8888) + async with asyncio.connect('127.0.0.1', 8888) as stream: + print(f'Send: {message!r}') + await stream.write(message.encode()) - print(f'Send: {message!r}') - await writer.write(message.encode()) - - data = await reader.read(100) - print(f'Received: {data.decode()!r}') - - print('Close the connection') - await writer.close() + data = await stream.read(100) + print(f'Received: {data.decode()!r}') asyncio.run(tcp_echo_client('Hello World!')) @@ -42,6 +37,32 @@ The following top-level asyncio functions can be used to create and work with streams: +.. coroutinefunction:: connect(host=None, port=None, \*, \ + limit=2**16, ssl=None, family=0, \ + proto=0, flags=0, sock=None, local_addr=None, \ + server_hostname=None, ssl_handshake_timeout=None, \ + happy_eyeballs_delay=None, interleave=None) + + Connect to TCP socket on *host* : *port* address and return a :class:`Stream` + object of mode :attr:`StreamMode.READWRITE`. + + + *limit* determines the buffer size limit used by the returned :class:`Stream` + instance. By default the *limit* is set to 64 KiB. + + The rest of the arguments are passed directly to :meth:`loop.create_connection`. + + The function can be used with ``await`` to get a connected stream:: + + stream = await asyncio.connect('127.0.0.1', 8888) + + The function can also be used as an async context manager:: + + async with asyncio.connect('127.0.0.1', 8888) as stream: + ... + + .. versionadded:: 3.8 + .. coroutinefunction:: open_connection(host=None, port=None, \*, \ loop=None, limit=None, ssl=None, family=0, \ proto=0, flags=0, sock=None, local_addr=None, \ @@ -69,10 +90,10 @@ and work with streams: .. deprecated-removed:: 3.8 3.10 - `open_connection()` is deprecated in favor of `connect()`. + `open_connection()` is deprecated in favor of :func:`connect`. .. coroutinefunction:: start_server(client_connected_cb, host=None, \ - port=None, \*, loop=None, limit=None, \ + port=None, \*, loop=None, limit=2**16, \ family=socket.AF_UNSPEC, \ flags=socket.AI_PASSIVE, sock=None, \ backlog=100, ssl=None, reuse_address=None, \ @@ -106,11 +127,58 @@ and work with streams: .. deprecated-removed:: 3.8 3.10 - `start_server()` is deprecated if favor of `StreamServer()` + `start_server()` is deprecated if favor of :class:`StreamServer` + +.. coroutinefunction:: connect_read_pipe(pipe, *, limit=2**16) + + Takes a :term:`file-like object ` *pipe* to return a + :class:`Stream` object of the mode :attr:`StreamMode.READ` that has + similar API of :class:`StreamReader`. It can also be used as an async context manager. + + *limit* determines the buffer size limit used by the returned :class:`Stream` + instance. By default the limit is set to 64 KiB. + .. versionadded:: 3.8 + +.. coroutinefunction:: connect_write_pipe(pipe, *, limit=2**16) + + Takes a :term:`file-like object ` *pipe* to return a + :class:`Stream` object of the mode :attr:`StreamMode.WRITE` that has + similar API of :class:`StreamWriter`. It can also be used as an async context manager. + + *limit* determines the buffer size limit used by the returned :class:`Stream` + instance. By default the limit is set to 64 KiB. + + .. versionadded:: 3.8 .. rubric:: Unix Sockets +.. function:: connect_unix(path=None, *, limit=2**16, ssl=None, \ + sock=None, server_hostname=None, \ + ssl_handshake_timeout=None) + + Establish a Unix socket connection to socket with *path* address and + return an awaitable :class:`Stream` object of the mode :attr:`StreamMode.READWRITE` + that can be used as a reader and a writer. + + *limit* determines the buffer size limit used by the returned :class:`Stream` + instance. By default the *limit* is set to 64 KiB. + + The rest of the arguments are passed directly to :meth:`loop.create_unix_connection`. + + The function can be used with ``await`` to get a connected stream:: + + stream = await asyncio.connect_unix('/tmp/example.sock') + + The function can also be used as an async context manager:: + + async with asyncio.connect_unix('/tmp/example.sock') as stream: + ... + + .. availability:: Unix. + + .. versionadded:: 3.8 + .. coroutinefunction:: open_unix_connection(path=None, \*, loop=None, \ limit=None, ssl=None, sock=None, \ server_hostname=None, ssl_handshake_timeout=None) @@ -134,7 +202,7 @@ and work with streams: .. deprecated-removed:: 3.8 3.10 - `open_unix_connection()` is deprecated if favor of `connect_unix()`. + ``open_unix_connection()`` is deprecated if favor of :func:`connect_unix`. .. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \ @@ -160,11 +228,176 @@ and work with streams: .. deprecated-removed:: 3.8 3.10 - `start_unix_server()` is deprecated in favor of `UnixStreamServer()`. + ``start_unix_server()`` is deprecated in favor of :class:`UnixStreamServer`. --------- +StreamServer +============ + +.. class:: StreamServer(client_connected_cb, /, host=None, port=None, *, \ + limit=2**16, family=socket.AF_UNSPEC, \ + flags=socket.AI_PASSIVE, sock=None, backlog=100, \ + ssl=None, reuse_address=None, reuse_port=None, \ + ssl_handshake_timeout=None, shutdown_timeout=60) + + The *client_connected_cb* callback is called whenever a new client + connection is established. It receives a :class:`Stream` object of the + mode :attr:`StreamMode.READWRITE`. + + *client_connected_cb* can be a plain callable or a + :ref:`coroutine function `; if it is a coroutine function, + it will be automatically scheduled as a :class:`Task`. + + *limit* determines the buffer size limit used by the + returned :class:`Stream` instance. By default the *limit* + is set to 64 KiB. + + The rest of the arguments are passed directly to + :meth:`loop.create_server`. + + .. coroutinemethod:: start_serving() + + Binds to the given host and port to start the server. + + .. coroutinemethod:: serve_forever() + + Start accepting connections until the coroutine is cancelled. + Cancellation of ``serve_forever`` task causes the server + to be closed. + + This method can be called if the server is already accepting + connections. Only one ``serve_forever`` task can exist per + one *Server* object. + + .. method:: is_serving() + + Returns ``True`` if the server is bound and currently serving. + + .. method:: bind() + + Bind the server to the given *host* and *port*. This method is + automatically called during ``__aenter__`` when :class:`StreamServer` is + used as an async context manager. + + .. method:: is_bound() + + Return ``True`` if the server is bound. + + .. coroutinemethod:: abort() + + Closes the connection and cancels all pending tasks. + + .. coroutinemethod:: close() + + Closes the connection. This method is automatically called during + ``__aexit__`` when :class:`StreamServer` is used as an async context + manager. + + .. attribute:: sockets + + Returns a tuple of socket objects the server is bound to. + + .. versionadded:: 3.8 + + +UnixStreamServer +================ + +.. class:: UnixStreamServer(client_connected_cb, /, path=None, *, \ + limit=2**16, sock=None, backlog=100, \ + ssl=None, ssl_handshake_timeout=None, shutdown_timeout=60) + + The *client_connected_cb* callback is called whenever a new client + connection is established. It receives a :class:`Stream` object of the + mode :attr:`StreamMode.READWRITE`. + + *client_connected_cb* can be a plain callable or a + :ref:`coroutine function `; if it is a coroutine function, + it will be automatically scheduled as a :class:`Task`. + + *limit* determines the buffer size limit used by the + returned :class:`Stream` instance. By default the *limit* + is set to 64 KiB. + + The rest of the arguments are passed directly to + :meth:`loop.create_unix_server`. + + .. coroutinemethod:: start_serving() + + Binds to the given host and port to start the server. + + .. method:: is_serving() + + Returns ``True`` if the server is bound and currently serving. + + .. method:: bind() + + Bind the server to the given *host* and *port*. This method is + automatically called during ``__aenter__`` when :class:`UnixStreamServer` is + used as an async context manager. + + .. method:: is_bound() + + Return ``True`` if the server is bound. + + .. coroutinemethod:: abort() + + Closes the connection and cancels all pending tasks. + + .. coroutinemethod:: close() + + Closes the connection. This method is automatically called during + ``__aexit__`` when :class:`UnixStreamServer` is used as an async context + manager. + + .. attribute:: sockets + + Returns a tuple of socket objects the server is bound to. + + .. availability:: Unix. + + .. versionadded:: 3.8 + +Stream +====== + +.. class:: Stream + + Represents a Stream object that provides APIs to read and write data + to the IO stream . It includes the API provided by :class:`StreamReader` + and :class:`StreamWriter`. + + Do not instantiate *Stream* objects directly; use API like :func:`connect` + and :class:`StreamServer` instead. + + .. versionadded:: 3.8 + + +StreamMode +========== + +.. class:: StreamMode + + A subclass of :class:`enum.Flag` that defines a set of values that can be + used to determine the ``mode`` of :class:`Stream` objects. + + .. data:: READ + + The stream object is readable and provides the API of :class:`StreamReader`. + + .. data:: WRITE + + The stream object is writeable and provides the API of :class:`StreamWriter`. + + .. data:: READWRITE + + The stream object is readable and writeable and provides the API of both + :class:`StreamReader` and :class:`StreamWriter`. + + .. versionadded:: 3.8 + StreamReader ============ @@ -366,22 +599,17 @@ Examples TCP echo client using streams ----------------------------- -TCP echo client using the :func:`asyncio.open_connection` function:: +TCP echo client using the :func:`asyncio.connect` function:: import asyncio async def tcp_echo_client(message): - reader, writer = await asyncio.open_connection( - '127.0.0.1', 8888) - - print(f'Send: {message!r}') - writer.write(message.encode()) + async with asyncio.connect('127.0.0.1', 8888) as stream: + print(f'Send: {message!r}') + await stream.write(message.encode()) - data = await reader.read(100) - print(f'Received: {data.decode()!r}') - - print('Close the connection') - writer.close() + data = await stream.read(100) + print(f'Received: {data.decode()!r}') asyncio.run(tcp_echo_client('Hello World!')) @@ -397,32 +625,28 @@ TCP echo client using the :func:`asyncio.open_connection` function:: TCP echo server using streams ----------------------------- -TCP echo server using the :func:`asyncio.start_server` function:: +TCP echo server using the :class:`asyncio.StreamServer` class:: import asyncio - async def handle_echo(reader, writer): - data = await reader.read(100) + async def handle_echo(stream): + data = await stream.read(100) message = data.decode() - addr = writer.get_extra_info('peername') + addr = stream.get_extra_info('peername') print(f"Received {message!r} from {addr!r}") print(f"Send: {message!r}") - writer.write(data) - await writer.drain() + await stream.write(data) print("Close the connection") - writer.close() + await stream.close() async def main(): - server = await asyncio.start_server( - handle_echo, '127.0.0.1', 8888) - - addr = server.sockets[0].getsockname() - print(f'Serving on {addr}') - - async with server: + async with asyncio.StreamServer( + handle_echo, '127.0.0.1', 8888) as server: + addr = server.sockets[0].getsockname() + print(f'Serving on {addr}') await server.serve_forever() asyncio.run(main()) @@ -446,11 +670,9 @@ Simple example querying HTTP headers of the URL passed on the command line:: async def print_http_headers(url): url = urllib.parse.urlsplit(url) if url.scheme == 'https': - reader, writer = await asyncio.open_connection( - url.hostname, 443, ssl=True) + stream = await asyncio.connect(url.hostname, 443, ssl=True) else: - reader, writer = await asyncio.open_connection( - url.hostname, 80) + stream = await asyncio.connect(url.hostname, 80) query = ( f"HEAD {url.path or '/'} HTTP/1.0\r\n" @@ -458,18 +680,14 @@ Simple example querying HTTP headers of the URL passed on the command line:: f"\r\n" ) - writer.write(query.encode('latin-1')) - while True: - line = await reader.readline() - if not line: - break - + stream.write(query.encode('latin-1')) + while (line := await stream.readline()): line = line.decode('latin1').rstrip() if line: print(f'HTTP header> {line}') # Ignore the body, close the socket - writer.close() + await stream.close() url = sys.argv[1] asyncio.run(print_http_headers(url)) @@ -490,7 +708,7 @@ Register an open socket to wait for data using streams ------------------------------------------------------ Coroutine waiting until a socket receives data using the -:func:`open_connection` function:: +:func:`asyncio.connect` function:: import asyncio import socket @@ -504,17 +722,15 @@ Coroutine waiting until a socket receives data using the rsock, wsock = socket.socketpair() # Register the open socket to wait for data. - reader, writer = await asyncio.open_connection(sock=rsock) - - # Simulate the reception of data from the network - loop.call_soon(wsock.send, 'abc'.encode()) + async with asyncio.connect(sock=rsock) as stream: + # Simulate the reception of data from the network + loop.call_soon(wsock.send, 'abc'.encode()) - # Wait for data - data = await reader.read(100) + # Wait for data + data = await stream.read(100) - # Got data, we are done: close the socket - print("Received:", data.decode()) - writer.close() + # Got data, we are done: close the socket + print("Received:", data.decode()) # Close the second socket wsock.close() diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 00dc66c48b2189..444fb6361b5eff 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -293,18 +293,26 @@ their completion. Subprocess and Threads ---------------------- -Standard asyncio event loop supports running subprocesses from -different threads, but there are limitations: +Standard asyncio event loop supports running subprocesses from different threads by +default. -* An event loop must run in the main thread. +On Windows subprocesses are provided by :class:`ProactorEventLoop` only (default), +:class:`SelectorEventLoop` has no subprocess support. -* The child watcher must be instantiated in the main thread - before executing subprocesses from other threads. Call the - :func:`get_child_watcher` function in the main thread to instantiate - the child watcher. +On UNIX *child watchers* are used for subprocess finish waiting, see +:ref:`asyncio-watchers` for more info. -Note that alternative event loop implementations might not share -the above limitations; please refer to their documentation. + +.. versionchanged:: 3.8 + + UNIX switched to use :class:`ThreadedChildWatcher` for spawning subprocesses from + different threads without any limitation. + + Spawning a subprocess with *inactive* current child watcher raises + :exc:`RuntimeError`. + +Note that alternative event loop implementations might have own limitations; +please refer to their documentation. .. seealso:: diff --git a/Doc/library/audit_events.rst b/Doc/library/audit_events.rst new file mode 100644 index 00000000000000..c23b9c618323de --- /dev/null +++ b/Doc/library/audit_events.rst @@ -0,0 +1,21 @@ +.. _audit-events: + +.. index:: single: audit events + +Audit events table +================== + +This table contains all events raised by :func:`sys.audit` or +:c:func:`PySys_Audit` calls throughout the CPython runtime and the +standard library. + +See :func:`sys.addaudithook` and :c:func:`PySys_AddAuditHook` for +information on handling these events. + +.. impl-detail:: + + This table is generated from the CPython documentation, and may not + represent events raised by other implementations. See your runtime + specific documentation for actual events raised. + +.. audit-event-table:: diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index bb5000a0736ccc..9ce5ca819c68a1 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -120,7 +120,7 @@ runtime. Public functions ---------------- -.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP) +.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` files along the way. Return a true value if all the files compiled successfully, @@ -185,10 +185,13 @@ Public functions .. versionchanged:: 3.7 The *invalidation_mode* parameter was added. + .. versionchanged:: 3.7.2 + The *invalidation_mode* parameter's default value is updated to None. + .. versionchanged:: 3.8 Setting *workers* to 0 now chooses the optimal number of cores. -.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP) +.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None) Compile the file with path *fullname*. Return a true value if the file compiled successfully, and a false value otherwise. @@ -232,7 +235,10 @@ Public functions .. versionchanged:: 3.7 The *invalidation_mode* parameter was added. -.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP) + .. versionchanged:: 3.7.2 + The *invalidation_mode* parameter's default value is updated to None. + +.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1, invalidation_mode=None) Byte-compile all the :file:`.py` files found along ``sys.path``. Return a true value if all the files compiled successfully, and a false value otherwise. @@ -255,6 +261,9 @@ Public functions .. versionchanged:: 3.7 The *invalidation_mode* parameter was added. + .. versionchanged:: 3.7.2 + The *invalidation_mode* parameter's default value is updated to None. + To force a recompile of all the :file:`.py` files in the :file:`Lib/` subdirectory and all its subdirectories:: diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index f2491dd24571be..41d47c7ef1530b 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -306,9 +306,10 @@ The :class:`Future` class encapsulates the asynchronous execution of a callable. .. method:: cancel() - Attempt to cancel the call. If the call is currently being executed and - cannot be cancelled then the method will return ``False``, otherwise the - call will be cancelled and the method will return ``True``. + Attempt to cancel the call. If the call is currently being executed or + finished running and cannot be cancelled then the method will return + ``False``, otherwise the call will be cancelled and the method will + return ``True``. .. method:: cancelled() @@ -423,8 +424,9 @@ Module Functions Wait for the :class:`Future` instances (possibly created by different :class:`Executor` instances) given by *fs* to complete. Returns a named 2-tuple of sets. The first set, named ``done``, contains the futures that - completed (finished or were cancelled) before the wait completed. The second - set, named ``not_done``, contains uncompleted futures. + completed (finished or cancelled futures) before the wait completed. The + second set, named ``not_done``, contains the futures that did not complete + (pending or running futures). *timeout* can be used to control the maximum number of seconds to wait before returning. *timeout* can be an int or float. If *timeout* is not specified @@ -455,7 +457,7 @@ Module Functions Returns an iterator over the :class:`Future` instances (possibly created by different :class:`Executor` instances) given by *fs* that yields futures as - they complete (finished or were cancelled). Any futures given by *fs* that + they complete (finished or cancelled futures). Any futures given by *fs* that are duplicated will be returned once. Any futures that completed before :func:`as_completed` is called will be yielded first. The returned iterator raises a :exc:`concurrent.futures.TimeoutError` if :meth:`~iterator.__next__` diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index c7bd89f9637226..a8e8bfb1e832bb 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -31,6 +31,7 @@ Interface summary: Raised for module specific errors. +.. _shallow_vs_deep_copy: The difference between shallow and deep copying is only relevant for compound objects (objects that contain other objects, like lists or class instances): diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 97172c588ae97f..eb0bcf4ae2e368 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -576,7 +576,7 @@ Here is a simple example of a POINT structure, which contains two integers named >>> POINT(1, 2, 3) Traceback (most recent call last): File "", line 1, in - ValueError: too many initializers + TypeError: too many initializers >>> You can, however, build much more complicated structures. A structure can @@ -1175,16 +1175,21 @@ Keep in mind that retrieving sub-objects from Structure, Unions, and Arrays doesn't *copy* the sub-object, instead it retrieves a wrapper object accessing the root-object's underlying buffer. -Another example that may behave different from what one would expect is this:: +Another example that may behave differently from what one would expect is this:: >>> s = c_char_p() - >>> s.value = "abc def ghi" + >>> s.value = b"abc def ghi" >>> s.value - 'abc def ghi' + b'abc def ghi' >>> s.value is s.value False >>> +.. note:: + + Objects instantiated from :class:`c_char_p` can only have their value set to bytes + or integers. + Why is it printing ``False``? ctypes instances are objects containing a memory block plus some :term:`descriptor`\s accessing the contents of the memory. Storing a Python object in the memory block does not store the object itself, @@ -1509,13 +1514,13 @@ object is available: :c:type:`int`, which is of course not always the truth, so you have to assign the correct :attr:`restype` attribute to use these functions. -.. audit-event:: ctypes.dlopen name +.. audit-event:: ctypes.dlopen name ctypes.LibraryLoader Loading a library through any of these objects raises an :ref:`auditing event ` ``ctypes.dlopen`` with string argument ``name``, the name used to load the library. -.. audit-event:: ctypes.dlsym "library name" +.. audit-event:: ctypes.dlsym library,name ctypes.LibraryLoader Accessing a function on a loaded library raises an auditing event ``ctypes.dlsym`` with arguments ``library`` (the library object) and ``name`` @@ -2043,10 +2048,10 @@ Data types This method returns a ctypes type instance using the memory specified by *address* which must be an integer. - .. audit-event:: ctypes.cdata address + .. audit-event:: ctypes.cdata address ctypes._CData.from_address This method, and others that indirectly call this method, raises an - :func:`auditing event ` ``ctypes.cdata`` with argument + :ref:`auditing event ` ``ctypes.cdata`` with argument ``address``. .. method:: from_param(obj) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index fb41aeeb5bed03..50f1b9e74949f0 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -401,7 +401,7 @@ systems. .. class:: date(year, month, day) - All arguments are required. Arguments may be integers, in the following + All arguments are required. Arguments must be integers in the following ranges: * ``MINYEAR <= year <= MAXYEAR`` @@ -714,7 +714,7 @@ Constructor: .. class:: datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0) The year, month and day arguments are required. *tzinfo* may be ``None``, or an - instance of a :class:`tzinfo` subclass. The remaining arguments may be integers, + instance of a :class:`tzinfo` subclass. The remaining arguments must be integers in the following ranges: * ``MINYEAR <= year <= MAXYEAR``, @@ -857,7 +857,9 @@ Other constructors, all class methods: .. caution:: This does not support parsing arbitrary ISO 8601 strings - it is only intended - as the inverse operation of :meth:`datetime.isoformat`. + as the inverse operation of :meth:`datetime.isoformat`. A more full-featured + ISO 8601 parser, ``dateutil.parser.isoparse`` is available in the third-party package + `dateutil `_. .. versionadded:: 3.7 @@ -1439,7 +1441,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. .. class:: time(hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0) All arguments are optional. *tzinfo* may be ``None``, or an instance of a - :class:`tzinfo` subclass. The remaining arguments may be integers, in the + :class:`tzinfo` subclass. The remaining arguments must be integers in the following ranges: * ``0 <= hour < 24``, @@ -2093,7 +2095,7 @@ format codes. | | where 0 is Sunday and 6 is | | | | | Saturday. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%d`` | Day of the month as a | 01, 02, ..., 31 | | +| ``%d`` | Day of the month as a | 01, 02, ..., 31 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | @@ -2106,29 +2108,29 @@ format codes. | | || Januar, Februar, ..., | | | | | Dezember (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | | +| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | \(9) | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%y`` | Year without century as a | 00, 01, ..., 99 | | +| ``%y`` | Year without century as a | 00, 01, ..., 99 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%Y`` | Year with century as a decimal | 0001, 0002, ..., 2013, | \(2) | | | number. | 2014, ..., 9998, 9999 | | +-----------+--------------------------------+------------------------+-------+ -| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | | +| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | | +| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | | | AM or PM. || am, pm (de_DE) | \(3) | +-----------+--------------------------------+------------------------+-------+ -| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | | +| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | \(9) | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4) | -| | decimal number. | | | +| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4), | +| | decimal number. | | \(9) | +-----------+--------------------------------+------------------------+-------+ | ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(5) | | | number, zero-padded on the | 999999 | | @@ -2142,19 +2144,19 @@ format codes. | ``%Z`` | Time zone name (empty string | (empty), UTC, EST, CST | | | | if the object is naive). | | | +-----------+--------------------------------+------------------------+-------+ -| ``%j`` | Day of the year as a | 001, 002, ..., 366 | | +| ``%j`` | Day of the year as a | 001, 002, ..., 366 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7) | -| | (Sunday as the first day of | | | +| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| | (Sunday as the first day of | | \(9) | | | the week) as a zero padded | | | | | decimal number. All days in a | | | | | new year preceding the first | | | | | Sunday are considered to be in | | | | | week 0. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7) | -| | (Monday as the first day of | | | +| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| | (Monday as the first day of | | \(9) | | | the week) as a decimal number. | | | | | All days in a new year | | | | | preceding the first Monday | | | @@ -2194,8 +2196,8 @@ incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. | ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | | | number where 1 is Monday. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8) | -| | number with Monday as | | | +| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8), | +| | number with Monday as | | \(9) | | | the first day of the week. | | | | | Week 01 is the week containing | | | | | Jan 4. | | | @@ -2291,6 +2293,11 @@ Notes: :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not interchangeable. +(9) + When used with the :meth:`strptime` method, the leading zero is optional + for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%J``, ``%U``, + ``%W``, and ``%V``. Format ``%y`` does require a leading zero. + .. rubric:: Footnotes .. [#] If, that is, we ignore the effects of Relativity diff --git a/Doc/library/debug.rst b/Doc/library/debug.rst index 88a2fa62a56588..60223657a44043 100644 --- a/Doc/library/debug.rst +++ b/Doc/library/debug.rst @@ -5,10 +5,13 @@ Debugging and Profiling These libraries help you with Python development: the debugger enables you to step through code, analyze stack frames and set breakpoints etc., and the profilers run code and give you a detailed breakdown of execution times, -allowing you to identify bottlenecks in your programs. +allowing you to identify bottlenecks in your programs. Auditing events +provide visibility into runtime behaviors that would otherwise require +intrusive debugging or patching. .. toctree:: + audit_events.rst bdb.rst faulthandler.rst pdb.rst diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index f044cb2d6e0a86..e245ab81cfb949 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -543,6 +543,16 @@ The :class:`SequenceMatcher` class has this constructor: to try :meth:`quick_ratio` or :meth:`real_quick_ratio` first to get an upper bound. + .. note:: + + Caution: The result of a :meth:`ratio` call may depend on the order of + the arguments. For instance:: + + >>> SequenceMatcher(None, 'tide', 'diet').ratio() + 0.25 + >>> SequenceMatcher(None, 'diet', 'tide').ratio() + 0.5 + .. method:: quick_ratio() diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 2a3ffb5e827119..39a3e130afd3e0 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -645,10 +645,12 @@ the original TOS1. .. opcode:: MAP_ADD (i) - Calls ``dict.setitem(TOS1[-i], TOS, TOS1)``. Used to implement dict + Calls ``dict.__setitem__(TOS1[-i], TOS1, TOS)``. Used to implement dict comprehensions. .. versionadded:: 3.1 + .. versionchanged:: 3.8 + Map value is TOS and map key is TOS1. Before, those were reversed. For all of the :opcode:`SET_ADD`, :opcode:`LIST_APPEND` and :opcode:`MAP_ADD` instructions, while the added value or key/value pair is popped off, the @@ -1219,10 +1221,10 @@ All of the following opcodes use their arguments. .. opcode:: EXTENDED_ARG (ext) - Prefixes any opcode which has an argument too big to fit into the default two - bytes. *ext* holds two additional bytes which, taken together with the - subsequent opcode's argument, comprise a four-byte argument, *ext* being the - two most-significant bytes. + Prefixes any opcode which has an argument too big to fit into the default one + byte. *ext* holds an additional byte which act as higher bits in the argument. + For each opcode, at most three prefixal ``EXTENDED_ARG`` are allowed, forming + an argument from two-byte to four-byte. .. opcode:: FORMAT_VALUE (flags) diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index c797f63326d1a2..a2bb045e57e3c0 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -119,6 +119,8 @@ Module API *verbosity* controls the level of output to :data:`sys.stdout` from the bootstrapping operation. + .. audit-event:: ensurepip.bootstrap root ensurepip.bootstrap + .. note:: The bootstrapping process has side effects on both ``sys.path`` and diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index b5a818e1cafa61..58e7126b0bf212 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -94,6 +94,13 @@ another rational number, or from a string. Denominator of the Fraction in lowest term. + .. method:: as_integer_ratio() + + Return a tuple of two integers, whose ratio is equal + to the Fraction and with a positive denominator. + + .. versionadded:: 3.8 + .. method:: from_float(flt) This class method constructs a :class:`Fraction` representing the exact diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 6c39f9a59fc1fd..e006d019b837dc 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -190,6 +190,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. *source_address* is a 2-tuple ``(host, port)`` for the socket to bind to as its source address before connecting. + .. audit-event:: ftplib.connect self,host,port ftplib.FTP.connect + .. versionchanged:: 3.3 *source_address* parameter was added. @@ -223,6 +225,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. Send a simple command string to the server and return the response string. + .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.sendcmd + .. method:: FTP.voidcmd(cmd) @@ -230,6 +234,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. nothing if a response code corresponding to success (codes in the range 200--299) is received. Raise :exc:`error_reply` otherwise. + .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.voidcmd + .. method:: FTP.retrbinary(cmd, callback, blocksize=8192, rest=None) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 415a65b4946f08..c748b086d8acae 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -128,6 +128,8 @@ are always available. They are listed here in alphabetical order. :func:`breakpoint` will automatically call that, allowing you to drop into the debugger of choice. + .. audit-event:: builtins.breakpoint breakpointhook breakpoint + .. versionadded:: 3.7 .. _func-bytearray: @@ -275,9 +277,9 @@ are always available. They are listed here in alphabetical order. If you want to parse Python code into its AST representation, see :func:`ast.parse`. - .. audit-event:: compile "source filename" + .. audit-event:: compile source,filename compile - Raises an :func:`auditing event ` ``compile`` with arguments + Raises an :ref:`auditing event ` ``compile`` with arguments ``source`` and ``filename``. This event may also be raised by implicit compilation. @@ -463,12 +465,16 @@ are always available. They are listed here in alphabetical order. dictionaries as global and local namespace. If the *globals* dictionary is present and does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is - inserted under that key before *expression* is parsed. - This means that *expression* normally has full - access to the standard :mod:`builtins` module and restricted environments are - propagated. If the *locals* dictionary is omitted it defaults to the *globals* - dictionary. If both dictionaries are omitted, the expression is executed in the - environment where :func:`eval` is called. The return value is the result of + inserted under that key before *expression* is parsed. This means that + *expression* normally has full access to the standard :mod:`builtins` + module and restricted environments are propagated. If the *locals* + dictionary is omitted it defaults to the *globals* dictionary. If both + dictionaries are omitted, the expression is executed with the *globals* and + *locals* in the environment where :func:`eval` is called. Note, *eval()* + does not have access to the :term:`nested scope`\s (non-locals) in the + enclosing environment. + + The return value is the result of the evaluated expression. Syntax errors are reported as exceptions. Example: >>> x = 1 @@ -488,10 +494,10 @@ are always available. They are listed here in alphabetical order. See :func:`ast.literal_eval` for a function that can safely evaluate strings with expressions containing only literals. - .. audit-event:: exec code_object + .. audit-event:: exec code_object eval - Raises an :func:`auditing event ` ``exec`` with the code object as - the argument. Code compilation events may also be raised. + Raises an :ref:`auditing event ` ``exec`` with the code object + as the argument. Code compilation events may also be raised. .. index:: builtin: exec @@ -523,10 +529,10 @@ are always available. They are listed here in alphabetical order. builtins are available to the executed code by inserting your own ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. - .. audit-event:: exec code_object + .. audit-event:: exec code_object exec - Raises an :func:`auditing event ` ``exec`` with the code object as - the argument. Code compilation events may also be raised. + Raises an :ref:`auditing event ` ``exec`` with the code object + as the argument. Code compilation events may also be raised. .. note:: @@ -777,12 +783,12 @@ are always available. They are listed here in alphabetical order. If the :mod:`readline` module was loaded, then :func:`input` will use it to provide elaborate line editing and history features. - .. audit-event:: builtins.input prompt + .. audit-event:: builtins.input prompt input - Raises an :func:`auditing event ` ``builtins.input`` with + Raises an :ref:`auditing event ` ``builtins.input`` with argument ``prompt`` before reading input - .. audit-event:: builtins.input/result result + .. audit-event:: builtins.input/result result input Raises an auditing event ``builtins.input/result`` with the result after successfully reading input. @@ -1220,7 +1226,7 @@ are always available. They are listed here in alphabetical order. (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`, and :mod:`shutil`. - .. audit-event:: open "file mode flags" + .. audit-event:: open file,mode,flags open The ``mode`` and ``flags`` arguments may have been modified or inferred from the original call. @@ -1562,11 +1568,11 @@ are always available. They are listed here in alphabetical order. about strings, see :ref:`textseq`. -.. function:: sum(iterable[, start]) +.. function:: sum(iterable, /, start=0) Sums *start* and the items of an *iterable* from left to right and returns the - total. *start* defaults to ``0``. The *iterable*'s items are normally numbers, - and the start value is not allowed to be a string. + total. The *iterable*'s items are normally numbers, and the start value is not + allowed to be a string. For some use cases, there are good alternatives to :func:`sum`. The preferred, fast way to concatenate a sequence of strings is by calling @@ -1581,10 +1587,17 @@ are always available. They are listed here in alphabetical order. Return a proxy object that delegates method calls to a parent or sibling class of *type*. This is useful for accessing inherited methods that have - been overridden in a class. The search order is same as that used by - :func:`getattr` except that the *type* itself is skipped. + been overridden in a class. + + The *object-or-type* determines the :term:`method resolution order` + to be searched. The search starts from the class right after the + *type*. + + For example, if :attr:`~class.__mro__` of *object-or-type* is + ``D -> B -> C -> A -> object`` and the value of *type* is ``B``, + then :func:`super` searches ``C -> A -> object``. - The :attr:`~class.__mro__` attribute of the *type* lists the method + The :attr:`~class.__mro__` attribute of the *object-or-type* lists the method resolution search order used by both :func:`getattr` and :func:`super`. The attribute is dynamic and can change whenever the inheritance hierarchy is updated. diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 2a5f0ddc4ef319..96584437cef16b 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -52,6 +52,8 @@ For example, ``'[?]'`` matches the character ``'?'``. more directories and subdirectories. If the pattern is followed by an ``os.sep``, only directories and subdirectories match. + .. audit-event:: glob.glob pathname,recursive glob.glob + .. note:: Using the "``**``" pattern in large directory trees may consume an inordinate amount of time. @@ -65,6 +67,8 @@ For example, ``'[?]'`` matches the character ``'?'``. Return an :term:`iterator` which yields the same values as :func:`glob` without actually storing them all simultaneously. + .. audit-event:: glob.glob pathname,recursive glob.iglob + .. function:: escape(pathname) diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index beaa720d732b4c..4e761cd39a0109 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -94,6 +94,11 @@ The module provides the following classes: :func:`ssl._create_unverified_context` can be passed to the *context* parameter. + .. versionchanged:: 3.8 + This class now enables TLS 1.3 + :attr:`ssl.SSLContext.post_handshake_auth` for the default *context* or + when *cert_file* is passed with a custom *context*. + .. deprecated:: 3.6 *key_file* and *cert_file* are deprecated in favor of *context*. diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index a9d7321a42c43f..1788bd8f833d1d 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -159,7 +159,7 @@ contained :class:`Cookie` objects. the :class:`CookieJar`'s :class:`CookiePolicy` instance are true and false respectively), the :mailheader:`Cookie2` header is also added when appropriate. - The *request* object (usually a :class:`urllib.request..Request` instance) + The *request* object (usually a :class:`urllib.request.Request` instance) must support the methods :meth:`get_full_url`, :meth:`get_host`, :meth:`get_type`, :meth:`unverifiable`, :meth:`has_header`, :meth:`get_header`, :meth:`header_items`, :meth:`add_unredirected_header` diff --git a/Doc/library/http.rst b/Doc/library/http.rst index 88d62cca3f9041..8df14578de1f1e 100644 --- a/Doc/library/http.rst +++ b/Doc/library/http.rst @@ -106,6 +106,7 @@ Code Enum Name Details ``428`` ``PRECONDITION_REQUIRED`` Additional HTTP Status Codes :rfc:`6585` ``429`` ``TOO_MANY_REQUESTS`` Additional HTTP Status Codes :rfc:`6585` ``431`` ``REQUEST_HEADER_FIELDS_TOO_LARGE`` Additional HTTP Status Codes :rfc:`6585` +``451`` ``UNAVAILABLE_FOR_LEGAL_REASONS`` An HTTP Status Code to Report Legal Obstacles :rfc:`7725` ``500`` ``INTERNAL_SERVER_ERROR`` HTTP/1.1 :rfc:`7231`, Section 6.6.1 ``501`` ``NOT_IMPLEMENTED`` HTTP/1.1 :rfc:`7231`, Section 6.6.2 ``502`` ``BAD_GATEWAY`` HTTP/1.1 :rfc:`7231`, Section 6.6.3 @@ -126,3 +127,6 @@ equal to the constant name (i.e. ``http.HTTPStatus.OK`` is also available as .. versionchanged:: 3.7 Added ``421 MISDIRECTED_REQUEST`` status code. + +.. versionadded:: 3.8 + Added ``451 UNAVAILABLE_FOR_LEGAL_REASONS`` status code. diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index bd24695c728233..0bd248c22b1820 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -207,18 +207,10 @@ Strip trailing whitespace Run menu (Editor window only) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python Shell - Open or wake up the Python Shell window. - -Check Module - Check the syntax of the module currently open in the Editor window. If the - module has not been saved IDLE will either prompt the user to save or - autosave, as selected in the General tab of the Idle Settings dialog. If - there is a syntax error, the approximate location is indicated in the - Editor window. +.. _run-module: Run Module - Do Check Module (above). If no error, restart the shell to clean the + Do :ref:`Check Module `. If no error, restart the shell to clean the environment, then execute the module. Output is displayed in the Shell window. Note that output requires use of ``print`` or ``write``. When execution is complete, the Shell retains focus and displays a prompt. @@ -226,6 +218,28 @@ Run Module This is similar to executing a file with ``python -i file`` at a command line. +.. _run-custom: + +Run... Customized + Same as :ref:`Run Module `, but run the module with customized + settings. *Command Line Arguments* extend :data:`sys.argv` as if passed + on a command line. The module can be run in the Shell without restarting. + +.. _check-module: + +Check Module + Check the syntax of the module currently open in the Editor window. If the + module has not been saved IDLE will either prompt the user to save or + autosave, as selected in the General tab of the Idle Settings dialog. If + there is a syntax error, the approximate location is indicated in the + Editor window. + +.. _python-shell: + +Python Shell + Open or wake up the Python Shell window. + + Shell menu (Shell window only) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -276,20 +290,32 @@ Options menu (Shell and Editor) Configure IDLE Open a configuration dialog and change preferences for the following: fonts, indentation, keybindings, text color themes, startup windows and - size, additional help sources, and extensions. On macOS, open the + size, additional help sources, and extensions. On macOS, open the configuration dialog by selecting Preferences in the application - menu. For more, see + menu. For more details, see :ref:`Setting preferences ` under Help and preferences. +Most configuration options apply to all windows or all future windows. +The option items below only apply to the active window. + Show/Hide Code Context (Editor Window only) Open a pane at the top of the edit window which shows the block context of the code which has scrolled above the top of the window. See - :ref:`Code Context ` in the Editing and Navigation section below. + :ref:`Code Context ` in the Editing and Navigation section + below. + +Show/Hide Line Numbers (Editor Window only) + Open a column to the left of the edit window which shows the number + of each line of text. The default is off, which may be changed in the + preferences (see :ref:`Setting preferences `). Zoom/Restore Height Toggles the window between normal size and maximum height. The initial size defaults to 40 lines by 80 chars unless changed on the General tab of the - Configure IDLE dialog. + Configure IDLE dialog. The maximum height for a screen is determined by + momentarily maximizing a window the first time one is zoomed on the screen. + Changing screen settings may invalidate the saved height. This toggle has + no effect when a window is maximized. Window menu (Shell and Editor) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -696,6 +722,10 @@ or ``print`` or ``write`` to sys.stdout or sys.stderr, IDLE should be started in a command line window. The secondary subprocess will then be attached to that window for input and output. +The IDLE code running in the execution process adds frames to the call stack +that would not be there otherwise. IDLE wraps ``sys.getrecursionlimit`` and +``sys.setrecursionlimit`` to reduce the effect of the additional stack frames. + If ``sys`` is reset by user code, such as with ``importlib.reload(sys)``, IDLE's changes are lost and input from the keyboard and output to the screen will not work correctly. diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index f027f82ddebe61..df63d820cfe043 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -361,6 +361,8 @@ An :class:`IMAP4` instance has the following methods: :meth:`IMAP4.send`, and :meth:`IMAP4.shutdown` methods. You may override this method. + .. audit-event:: imaplib.open self,host,port imaplib.IMAP4.open + .. method:: IMAP4.partial(message_num, message_part, start, length) @@ -430,6 +432,8 @@ An :class:`IMAP4` instance has the following methods: Sends ``data`` to the remote server. You may override this method. + .. audit-event:: imaplib.send self,data imaplib.IMAP4.send + .. method:: IMAP4.setacl(mailbox, who, what) diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 2126498e728cfa..23c8cdeb21c25d 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -158,6 +158,13 @@ Once you have the file, you can also read its contents:: return s.encode('utf-8') return s +In the case where the metadata file listing files +(RECORD or SOURCES.txt) is missing, ``files()`` will +return ``None``. The caller may wish to wrap calls to +``files()`` in `always_iterable +`_ +or otherwise guard against this condition if the target +distribution is not known to have the metadata present. .. _requirements: diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 23831c75842f1f..cb8360caca5e32 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -500,7 +500,7 @@ ABC hierarchy:: packages or a module). Loaders that wish to support resource reading are expected to - provide a method called ``get_resource_loader(fullname)`` which + provide a method called ``get_resource_reader(fullname)`` which returns an object implementing this ABC's interface. If the module specified by fullname is not a package, this method should return :const:`None`. An object compatible with this ABC should only be @@ -1379,8 +1379,8 @@ an :term:`importer`. bytecode file. An empty string represents no optimization, so ``/foo/bar/baz.py`` with an *optimization* of ``''`` will result in a bytecode path of ``/foo/bar/__pycache__/baz.cpython-32.pyc``. ``None`` causes - the interpter's optimization level to be used. Any other value's string - representation being used, so ``/foo/bar/baz.py`` with an *optimization* of + the interpreter's optimization level to be used. Any other value's string + representation is used, so ``/foo/bar/baz.py`` with an *optimization* of ``2`` will lead to the bytecode path of ``/foo/bar/__pycache__/baz.cpython-32.opt-2.pyc``. The string representation of *optimization* can only be alphanumeric, else :exc:`ValueError` is raised. @@ -1638,15 +1638,16 @@ import, then you should use :func:`importlib.util.find_spec`. # For illustrative purposes. name = 'itertools' - spec = importlib.util.find_spec(name) - if spec is None: - print("can't find the itertools module") - else: + if name in sys.modules: + print(f"{name!r} already in sys.modules") + elif (spec := importlib.util.find_spec(name)) is not None: # If you chose to perform the actual import ... module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - # Adding the module to sys.modules is optional. sys.modules[name] = module + spec.loader.exec_module(module) + print(f"{name!r} has been imported") + else: + print(f"can't find the {name!r} module") Importing a source file directly @@ -1665,10 +1666,9 @@ To import a Python source file directly, use the following recipe spec = importlib.util.spec_from_file_location(module_name, file_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - # Optional; only necessary if you want to be able to import the module - # by name later. sys.modules[module_name] = module + spec.loader.exec_module(module) + Setting up an importer @@ -1740,8 +1740,8 @@ Python 3.6 and newer for other parts of the code). msg = f'No module named {absolute_name!r}' raise ModuleNotFoundError(msg, name=absolute_name) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) sys.modules[absolute_name] = module + spec.loader.exec_module(module) if path is not None: setattr(parent_module, child_name, module) return module diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 2fb27c3aad78b9..70e01153d41966 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -120,9 +120,9 @@ High-level Module Interface This is an alias for the builtin :func:`open` function. - .. audit-event:: open "path mode flags" + .. audit-event:: open path,mode,flags io.open - This function raises an :func:`auditing event ` ``open`` with + This function raises an :ref:`auditing event ` ``open`` with arguments ``path``, ``mode`` and ``flags``. The ``mode`` and ``flags`` arguments may have been modified or inferred from the original call. @@ -327,7 +327,7 @@ I/O Base Classes Note that it's already possible to iterate on file objects using ``for line in file: ...`` without calling ``file.readlines()``. - .. method:: seek(offset[, whence]) + .. method:: seek(offset, whence=SEEK_SET) Change the stream position to the given byte *offset*. *offset* is interpreted relative to the position indicated by *whence*. The default @@ -831,7 +831,7 @@ Text I/O If *size* is specified, at most *size* characters will be read. - .. method:: seek(offset[, whence]) + .. method:: seek(offset, whence=SEEK_SET) Change the stream position to the given *offset*. Behaviour depends on the *whence* parameter. The default value for *whence* is @@ -945,7 +945,7 @@ Text I/O *errors*, *newline*, *line_buffering* and *write_through*. Parameters not specified keep current settings, except - ``errors='strict`` is used when *encoding* is specified but + ``errors='strict'`` is used when *encoding* is specified but *errors* is not specified. It is not possible to change the encoding or newline if some data diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index b3a0a5f5192da1..8d134d43806593 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -645,6 +645,10 @@ loops that truncate the stream. used anywhere else; otherwise, the *iterable* could get advanced without the tee objects being informed. + ``tee`` iterators are not threadsafe. A :exc:`RuntimeError` may be + raised when using simultaneously iterators returned by the same :func:`tee` + call, even if the original *iterable* is threadsafe. + This itertool may require significant auxiliary storage (depending on how much temporary data needs to be stored). In general, if one iterator uses most or all of the data before another iterator starts, it is faster to use @@ -691,6 +695,12 @@ Itertools Recipes This section shows recipes for creating an extended toolset using the existing itertools as building blocks. +Substantially all of these recipes and many, many others can be installed from +the `more-itertools project `_ found +on the Python Package Index:: + + pip install more-itertools + The extended tools offer the same high performance as the underlying toolset. The superior memory performance is kept by processing elements one at a time rather than bringing the whole iterable into memory all at once. Code volume is @@ -913,9 +923,3 @@ which incur interpreter overhead. result.append(pool[-1-n]) return tuple(result) -Note, many of the above recipes can be optimized by replacing global lookups -with local variables defined as default values. For example, the -*dotproduct* recipe can be written as:: - - def dotproduct(vec1, vec2, sum=sum, map=map, mul=operator.mul): - return sum(map(mul, vec1, vec2)) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index b476c372370d06..23e39e95f783ef 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -120,6 +120,18 @@ See :ref:`json-commandline` for detailed documentation. value) is also a subset of YAML 1.0 and 1.1. This module can thus also be used as a YAML serializer. +.. note:: + + This module's encoders and decoders preserve input and output order by + default. Order is only lost if the underlying containers are unordered. + + Prior to Python 3.7, :class:`dict` was not guaranteed to be ordered, so + inputs and outputs were typically scrambled unless + :class:`collections.OrderedDict` was specifically requested. Starting + with Python 3.7, the regular :class:`dict` became order preserving, so + it is no longer necessary to specify :class:`collections.OrderedDict` for + JSON generation and parsing. + Basic Usage ----------- diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index dee9a84e333764..32919c1a2e2909 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -470,7 +470,12 @@ sends logging output to a network socket. The base class uses a TCP socket. .. method:: makePickle(record) Pickles the record's attribute dictionary in binary format with a length - prefix, and returns it ready for transmission across the socket. + prefix, and returns it ready for transmission across the socket. The + details of this operation are equivalent to:: + + data = pickle.dumps(record_attr_dict, 1) + datalen = struct.pack('>L', len(data)) + return datalen + data Note that pickles aren't completely secure. If you are concerned about security, you may want to override this method to implement a more secure @@ -481,8 +486,12 @@ sends logging output to a network socket. The base class uses a TCP socket. .. method:: send(packet) - Send a pickled string *packet* to the socket. This function allows for - partial sends which can happen when the network is busy. + Send a pickled byte-string *packet* to the socket. The format of the sent + byte-string is as described in the documentation for + :meth:`~SocketHandler.makePickle`. + + This function allows for partial sends, which can happen when the network + is busy. .. method:: createSocket() @@ -543,7 +552,8 @@ over UDP sockets. .. method:: send(s) - Send a pickled string to a socket. + Send a pickled byte-string to a socket. The format of the sent byte-string + is as described in the documentation for :meth:`SocketHandler.makePickle`. .. _syslog-handler: @@ -830,7 +840,8 @@ should, then :meth:`flush` is expected to do the flushing. .. class:: BufferingHandler(capacity) - Initializes the handler with a buffer of the specified capacity. + Initializes the handler with a buffer of the specified capacity. Here, + *capacity* means the number of logging records buffered. .. method:: emit(record) @@ -854,12 +865,13 @@ should, then :meth:`flush` is expected to do the flushing. .. class:: MemoryHandler(capacity, flushLevel=ERROR, target=None, flushOnClose=True) Returns a new instance of the :class:`MemoryHandler` class. The instance is - initialized with a buffer size of *capacity*. If *flushLevel* is not specified, - :const:`ERROR` is used. If no *target* is specified, the target will need to be - set using :meth:`setTarget` before this handler does anything useful. If - *flushOnClose* is specified as ``False``, then the buffer is *not* flushed when - the handler is closed. If not specified or specified as ``True``, the previous - behaviour of flushing the buffer will occur when the handler is closed. + initialized with a buffer size of *capacity* (number of records buffered). + If *flushLevel* is not specified, :const:`ERROR` is used. If no *target* is + specified, the target will need to be set using :meth:`setTarget` before this + handler does anything useful. If *flushOnClose* is specified as ``False``, + then the buffer is *not* flushed when the handler is closed. If not specified + or specified as ``True``, the previous behaviour of flushing the buffer will + occur when the handler is closed. .. versionchanged:: 3.6 The *flushOnClose* parameter was added. @@ -959,14 +971,21 @@ possible, while any potentially slow operations (such as sending an email via .. class:: QueueHandler(queue) Returns a new instance of the :class:`QueueHandler` class. The instance is - initialized with the queue to send messages to. The queue can be any - queue-like object; it's used as-is by the :meth:`enqueue` method, which needs - to know how to send messages to it. + initialized with the queue to send messages to. The *queue* can be any + queue-like object; it's used as-is by the :meth:`enqueue` method, which + needs to know how to send messages to it. The queue is not *required* to + have the task tracking API, which means that you can use + :class:`~queue.SimpleQueue` instances for *queue*. .. method:: emit(record) - Enqueues the result of preparing the LogRecord. + Enqueues the result of preparing the LogRecord. Should an exception + occur (e.g. because a bounded queue has filled up), the + :meth:`~logging.Handler.handleError` method is called to handle the + error. This can result in the record silently being dropped (if + :attr:`logging.raiseExceptions` is ``False``) or a message printed to + ``sys.stderr`` (if :attr:`logging.raiseExceptions` is ``True``). .. method:: prepare(record) @@ -1017,11 +1036,14 @@ possible, while any potentially slow operations (such as sending an email via initialized with the queue to send messages to and a list of handlers which will handle entries placed on the queue. The queue can be any queue-like object; it's passed as-is to the :meth:`dequeue` method, which needs - to know how to get messages from it. If ``respect_handler_level`` is ``True``, - a handler's level is respected (compared with the level for the message) when - deciding whether to pass messages to that handler; otherwise, the behaviour - is as in previous Python versions - to always pass each message to each - handler. + to know how to get messages from it. The queue is not *required* to have the + task tracking API (though it's used if available), which means that you can + use :class:`~queue.SimpleQueue` instances for *queue*. + + If ``respect_handler_level`` is ``True``, a handler's level is respected + (compared with the level for the message) when deciding whether to pass + messages to that handler; otherwise, the behaviour is as in previous Python + versions - to always pass each message to each handler. .. versionchanged:: 3.5 The ``respect_handler_levels`` argument was added. diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 08555c3a357602..8ea00a92fb7346 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -50,8 +50,8 @@ listed below. Logger Objects -------------- -Loggers have the following attributes and methods. Note that Loggers are never -instantiated directly, but always through the module-level function +Loggers have the following attributes and methods. Note that Loggers should +*NEVER* be instantiated directly, but always through the module-level function ``logging.getLogger(name)``. Multiple calls to :func:`getLogger` with the same name will always return a reference to the same Logger object. @@ -1226,7 +1226,9 @@ functions. The class should define :meth:`__init__` such that only a name argument is required, and the :meth:`__init__` should call :meth:`Logger.__init__`. This function is typically called before any loggers are instantiated by applications - which need to use custom logger behavior. + which need to use custom logger behavior. After this call, as at any other + time, do not instantiate loggers directly using the subclass: continue to use + the :func:`logging.getLogger` API to get your loggers. .. function:: setLogRecordFactory(factory) diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index d901ad2cc1c48e..f82a3b200deb7c 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -308,6 +308,9 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. representation. If *create* is ``True``, the mailbox is created if it does not exist. + If *create* is ``True`` and the *dirname* path exists, it will be treated as + an existing maildir without attempting to verify its directory layout. + It is for historical reasons that *dirname* is named as such rather than *path*. Maildir is a directory-based mailbox format invented for the qmail mail diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 28ed5d21f03adb..776768dd8ee7c6 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -210,7 +210,7 @@ Number-theoretic and representation functions of *x* and are floats. -.. function:: perm(n, k) +.. function:: perm(n, k=None) Return the number of ways to choose *k* items from *n* items without repetition and with order. @@ -218,6 +218,9 @@ Number-theoretic and representation functions Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates to zero when ``k > n``. + If *k* is not specified or is None, then *k* defaults to *n* + and the function returns ``n!``. + Raises :exc:`TypeError` if either of the arguments are not integers. Raises :exc:`ValueError` if either of the arguments are negative. @@ -391,7 +394,8 @@ Trigonometric functions .. function:: dist(p, q) Return the Euclidean distance between two points *p* and *q*, each - given as a tuple of coordinates. The two tuples must be the same size. + given as a sequence (or iterable) of coordinates. The two points + must have the same dimension. Roughly equivalent to:: diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 5728407cb34c48..f610032acbe417 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -93,6 +93,10 @@ behavior of the module. Specifying an empty list for *files* will prevent the system defaults from being applied: only the well-known values will be present from a built-in list. + If *files* is ``None`` the internal data structure is completely rebuilt to its + initial default value. This is a stable operation and will produce the same results + when called multiple times. + .. versionchanged:: 3.2 Previously, Windows registry settings were ignored. diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index c7a13abad888d9..12b14d69332d72 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -67,7 +67,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length will be relative to the offset from the beginning of the file. *offset* defaults to 0. *offset* must be a multiple of the :const:`ALLOCATIONGRANULARITY`. - .. audit-event:: mmap.__new__ "fileno length access offset" + .. audit-event:: mmap.__new__ fileno,length,access,offset mmap.mmap .. class:: mmap(fileno, length, flags=MAP_SHARED, prot=PROT_WRITE|PROT_READ, access=ACCESS_DEFAULT[, offset]) :noindex: @@ -156,7 +156,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length mm.close() - .. audit-event:: mmap.__new__ "fileno length access offset" + .. audit-event:: mmap.__new__ fileno,length,access,offset mmap.mmap Memory-mapped file objects support the following methods: diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index a4771d3a84cd40..d8182feab963a1 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -126,8 +126,9 @@ to start a process. These *start methods* are .. versionchanged:: 3.8 - On macOS, *spawn* start method is now the default: *fork* start method is no - longer reliable on macOS, see :issue:`33725`. + On macOS, the *spawn* start method is now the default. The *fork* start + method should be considered unsafe as it can lead to crashes of the + subprocess. See :issue:`33725`. .. versionchanged:: 3.4 *spawn* added on all unix platforms, and *forkserver* added for @@ -2279,6 +2280,10 @@ with the :class:`Pool` class. Return whether the call completed without raising an exception. Will raise :exc:`AssertionError` if the result is not ready. + .. versionchanged:: 3.7 + If the result is not ready, :exc:`ValueError` is raised instead of + :exc:`AssertionError`. + The following example demonstrates the use of a pool:: from multiprocessing import Pool diff --git a/Doc/library/nntplib.rst b/Doc/library/nntplib.rst index 56188c7ef53880..46f1c0783551c2 100644 --- a/Doc/library/nntplib.rst +++ b/Doc/library/nntplib.rst @@ -79,6 +79,13 @@ The module itself defines the following classes: ('211 1755 1 1755 gmane.comp.python.committers', 1755, 1, 1755, 'gmane.comp.python.committers') >>> + .. audit-event:: nntplib.connect self,host,port nntplib.NNTP + + .. audit-event:: nntplib.putline self,line nntplib.NNTP + + All commands will raise an :ref:`auditing event ` + ``nntplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. .. versionchanged:: 3.2 *usenetrc* is now ``False`` by default. @@ -100,6 +107,14 @@ The module itself defines the following classes: STARTTLS as described below. However, some servers only support the former. + .. audit-event:: nntplib.connect self,host,port nntplib.NNTP_SSL + + .. audit-event:: nntplib.putline self,line nntplib.NNTP_SSL + + All commands will raise an :ref:`auditing event ` + ``nntplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. + .. versionadded:: 3.2 .. versionchanged:: 3.4 diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 8e7ee8bfe7845f..640ceecbc78f16 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -87,9 +87,10 @@ the :mod:`glob` module.) .. function:: commonpath(paths) Return the longest common sub-path of each pathname in the sequence - *paths*. Raise :exc:`ValueError` if *paths* contains both absolute and relative - pathnames, or if *paths* is empty. Unlike :func:`commonprefix`, this - returns a valid path. + *paths*. Raise :exc:`ValueError` if *paths* contain both absolute + and relative pathnames, the *paths* are on the different drives or + if *paths* is empty. Unlike :func:`commonprefix`, this returns a + valid path. .. availability:: Unix, Windows. @@ -349,11 +350,19 @@ the :mod:`glob` module.) .. function:: realpath(path) Return the canonical path of the specified filename, eliminating any symbolic - links encountered in the path (if they are supported by the operating system). + links encountered in the path (if they are supported by the operating + system). + + .. note:: + When symbolic link cycles occur, the returned path will be one member of + the cycle, but no guarantee is made about which member that will be. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.8 + Symbolic links and junctions are now resolved on Windows. + .. function:: relpath(path, start=os.curdir) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 107764ba4d539e..67fe36b54e2f87 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -36,9 +36,9 @@ Notes on the availability of these functions: .. note:: - All functions in this module raise :exc:`OSError` in the case of invalid or - inaccessible file names and paths, or other arguments that have the correct - type, but are not accepted by the operating system. + All functions in this module raise :exc:`OSError` (or subclasses thereof) in + the case of invalid or inaccessible file names and paths, or other arguments + that have the correct type, but are not accepted by the operating system. .. exception:: error @@ -851,7 +851,7 @@ as internal buffering of data. most *length* bytes in size. As of Python 3.3, this is equivalent to ``os.truncate(fd, length)``. - .. audit-event:: os.truncate "fd length" + .. audit-event:: os.truncate fd,length os.ftruncate .. availability:: Unix, Windows. @@ -938,7 +938,7 @@ as internal buffering of data. This function can support :ref:`paths relative to directory descriptors ` with the *dir_fd* parameter. - .. audit-event:: open "path mode flags" + .. audit-event:: open path,mode,flags os.open .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. @@ -1599,6 +1599,9 @@ features: This function can support :ref:`specifying a file descriptor `. The descriptor must refer to an opened directory, not an open file. + This function can raise :exc:`OSError` and subclasses such as + :exc:`FileNotFoundError`, :exc:`PermissionError`, and :exc:`NotADirectoryError`. + .. versionadded:: 3.3 Added support for specifying *path* as a file descriptor on some platforms. @@ -1730,6 +1733,11 @@ features: Return a bytestring representing the current working directory. + .. versionchanged:: 3.8 + The function now uses the UTF-8 encoding on Windows, rather than the ANSI + code page: see :pep:`529` for the rationale. The function is no longer + deprecated on Windows. + .. function:: lchflags(path, flags) @@ -1801,6 +1809,8 @@ features: This function can also support :ref:`specifying a file descriptor `; the file descriptor must refer to a directory. + .. audit-event:: os.listdir path os.listdir + .. note:: To encode ``str`` filenames to ``bytes``, use :func:`~os.fsencode`. @@ -1848,6 +1858,12 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *src* and *dst*. + .. versionchanged:: 3.8 + On Windows, now opens reparse points that represent another path + (name surrogates), including symbolic links and directory junctions. + Other kinds of reparse points are resolved by the operating system as + for :func:`~os.stat`. + .. function:: mkdir(path, mode=0o777, *, dir_fd=None) @@ -1891,8 +1907,8 @@ features: directories you can set the umask before invoking :func:`makedirs`. The file permission bits of existing parent directories are not changed. - If *exist_ok* is ``False`` (the default), an :exc:`OSError` is raised if the - target directory already exists. + If *exist_ok* is ``False`` (the default), an :exc:`FileExistsError` is + raised if the target directory already exists. .. note:: @@ -2029,6 +2045,10 @@ features: This function can also support :ref:`paths relative to directory descriptors `. + When trying to resolve a path that may contain links, use + :func:`~os.path.realpath` to properly handle recursion and platform + differences. + .. availability:: Unix, Windows. .. versionchanged:: 3.2 @@ -2043,10 +2063,15 @@ features: .. versionchanged:: 3.8 Accepts a :term:`path-like object` and a bytes object on Windows. + .. versionchanged:: 3.8 + Added support for directory junctions, and changed to return the + substitution path (which typically includes ``\\?\`` prefix) rather + than the optional "print name" field that was previously returned. + .. function:: remove(path, *, dir_fd=None) - Remove (delete) the file *path*. If *path* is a directory, :exc:`OSError` is - raised. Use :func:`rmdir` to remove directories. + Remove (delete) the file *path*. If *path* is a directory, an + :exc:`IsADirectoryError` is raised. Use :func:`rmdir` to remove directories. This function can support :ref:`paths relative to directory descriptors `. @@ -2083,13 +2108,19 @@ features: .. function:: rename(src, dst, *, src_dir_fd=None, dst_dir_fd=None) - Rename the file or directory *src* to *dst*. If *dst* is a directory, - :exc:`OSError` will be raised. On Unix, if *dst* exists and is a file, it will - be replaced silently if the user has permission. The operation may fail on some - Unix flavors if *src* and *dst* are on different filesystems. If successful, - the renaming will be an atomic operation (this is a POSIX requirement). On - Windows, if *dst* already exists, :exc:`OSError` will be raised even if it is a - file. + Rename the file or directory *src* to *dst*. If *dst* exists, the operation + will fail with an :exc:`OSError` subclass in a number of cases: + + On Windows, if *dst* exists a :exc:`FileExistsError` is always raised. + + On Unix, if *src* is a file and *dst* is a directory or vice-versa, an + :exc:`IsADirectoryError` or a :exc:`NotADirectoryError` will be raised + respectively. If both are directories and *dst* is empty, *dst* will be + silently replaced. If *dst* is a non-empty directory, an :exc:`OSError` + is raised. If both are files, *dst* it will be replaced silently if the user + has permission. The operation may fail on some Unix flavors if *src* and + *dst* are on different filesystems. If successful, the renaming will be an + atomic operation (this is a POSIX requirement). This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to supply :ref:`paths relative to directory descriptors `. @@ -2138,9 +2169,10 @@ features: .. function:: rmdir(path, *, dir_fd=None) - Remove (delete) the directory *path*. Only works when the directory is - empty, otherwise, :exc:`OSError` is raised. In order to remove whole - directory trees, :func:`shutil.rmtree` can be used. + Remove (delete) the directory *path*. If the directory does not exist or is + not empty, an :exc:`FileNotFoundError` or an :exc:`OSError` is raised + respectively. In order to remove whole directory trees, + :func:`shutil.rmtree` can be used. This function can support :ref:`paths relative to directory descriptors `. @@ -2178,6 +2210,8 @@ features: This function can also support :ref:`specifying a file descriptor `; the file descriptor must refer to a directory. + .. audit-event:: os.scandir path os.scandir + The :func:`scandir` iterator supports the :term:`context manager` protocol and has the following method: @@ -2347,7 +2381,8 @@ features: On Unix, this method always requires a system call. On Windows, it only requires a system call if *follow_symlinks* is ``True`` and the - entry is a symbolic link. + entry is a reparse point (for example, a symbolic link or directory + junction). On Windows, the ``st_ino``, ``st_dev`` and ``st_nlink`` attributes of the :class:`stat_result` are always set to zero. Call :func:`os.stat` to @@ -2384,6 +2419,17 @@ features: This function can support :ref:`specifying a file descriptor ` and :ref:`not following symlinks `. + On Windows, passing ``follow_symlinks=False`` will disable following all + name-surrogate reparse points, which includes symlinks and directory + junctions. Other types of reparse points that do not resemble links or that + the operating system is unable to follow will be opened directly. When + following a chain of multiple links, this may result in the original link + being returned instead of the non-link that prevented full traversal. To + obtain stat results for the final path in this case, use the + :func:`os.path.realpath` function to resolve the path name as far as + possible and call :func:`lstat` on the result. This does not apply to + dangling symlinks or junction points, which will raise the usual exceptions. + .. index:: module: stat Example:: @@ -2408,6 +2454,14 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.8 + On Windows, all reparse points that can be resolved by the operating + system are now followed, and passing ``follow_symlinks=False`` + disables following all name surrogate reparse points. If the operating + system reaches a reparse point that it is not able to follow, *stat* now + returns the information for the original path as if + ``follow_symlinks=False`` had been specified instead of raising an error. + .. class:: stat_result @@ -2559,7 +2613,7 @@ features: File type. - On Windows systems, the following attribute is also available: + On Windows systems, the following attributes are also available: .. attribute:: st_file_attributes @@ -2568,6 +2622,12 @@ features: :c:func:`GetFileInformationByHandle`. See the ``FILE_ATTRIBUTE_*`` constants in the :mod:`stat` module. + .. attribute:: st_reparse_tag + + When :attr:`st_file_attributes` has the ``FILE_ATTRIBUTE_REPARSE_POINT`` + set, this field contains the tag identifying the type of reparse point. + See the ``IO_REPARSE_TAG_*`` constants in the :mod:`stat` module. + The standard module :mod:`stat` defines functions and constants that are useful for extracting information from a :c:type:`stat` structure. (On Windows, some items are filled with dummy values.) @@ -2595,6 +2655,14 @@ features: .. versionadded:: 3.7 Added the :attr:`st_fstype` member to Solaris/derivatives. + .. versionadded:: 3.8 + Added the :attr:`st_reparse_tag` member on Windows. + + .. versionchanged:: 3.8 + On Windows, the :attr:`st_mode` member now identifies special + files as :const:`S_IFCHR`, :const:`S_IFIFO` or :const:`S_IFBLK` + as appropriate. + .. function:: statvfs(path) Perform a :c:func:`statvfs` system call on the given path. The return value is @@ -2784,7 +2852,7 @@ features: This function can support :ref:`specifying a file descriptor `. - .. audit-event:: os.truncate "path length" + .. audit-event:: os.truncate path,length os.truncate .. availability:: Unix, Windows. @@ -3011,6 +3079,13 @@ features: (or a bitwise ORed combination of them). By default, the new file descriptor is :ref:`non-inheritable `. + The name supplied in *name* is used as a filename and will be displayed as + the target of the corresponding symbolic link in the directory + ``/proc/self/fd/``. The displayed name is always prefixed with ``memfd:`` + and serves only for debugging purposes. Names do not affect the behavior of + the file descriptor, and as such multiple files can have the same name + without any side effects. + .. availability:: Linux 3.17 or newer with glibc 2.27 or newer. .. versionadded:: 3.8 @@ -3783,7 +3858,7 @@ written in Python, such as a mail server's external command delivery program. to using this function. See the :ref:`subprocess-replacements` section in the :mod:`subprocess` documentation for some helpful recipes. - .. audit-event:: os.system command + .. audit-event:: os.system command os.system .. availability:: Unix, Windows. @@ -3804,7 +3879,9 @@ written in Python, such as a mail server's external command delivery program. :attr:`children_system`, and :attr:`elapsed` in that order. See the Unix manual page - :manpage:`times(2)` or the corresponding Windows Platform API documentation. + :manpage:`times(2)` and :manpage:`times(3)` manual page on Unix or `the GetProcessTimes MSDN + ` + _ on Windows. On Windows, only :attr:`user` and :attr:`system` are known; the other attributes are zero. diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index c7864e9e3f2225..f26b6a8b553b36 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -181,6 +181,8 @@ access further features, you have to do this yourself: import pdb; pdb.Pdb(skip=['django.*']).set_trace() + .. audit-event:: pdb.Pdb "" pdb.Pdb + .. versionadded:: 3.1 The *skip* argument. diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 6aa30492c7060c..9442efa2b667fc 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -30,9 +30,17 @@ avoid confusion, the terms used here are "pickling" and "unpickling". .. warning:: - The :mod:`pickle` module is not secure against erroneous or maliciously - constructed data. Never unpickle data received from an untrusted or - unauthenticated source. + The ``pickle`` module **is not secure**. Only unpickle data you trust. + + It is possible to construct malicious pickle data which will **execute + arbitrary code during unpickling**. Never unpickle data that could have come + from an untrusted source, or that could have been tampered with. + + Consider signing data with :mod:`hmac` if you need to ensure that it has not + been tampered with. + + Safer serialization formats such as :mod:`json` may be more appropriate if + you are processing untrusted data. See :ref:`comparison-with-json`. Relationship to other Python modules @@ -75,6 +83,9 @@ The :mod:`pickle` module differs from :mod:`marshal` in several significant ways pickling and unpickling code deals with Python 2 to Python 3 type differences if your data is crossing that unique breaking change language boundary. + +.. _comparison-with-json: + Comparison with ``json`` ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -94,7 +105,10 @@ There are fundamental differences between the pickle protocols and types, and no custom classes; pickle can represent an extremely large number of Python types (many of them automatically, by clever usage of Python's introspection facilities; complex cases can be tackled by - implementing :ref:`specific object APIs `). + implementing :ref:`specific object APIs `); + +* Unlike pickle, deserializing untrusted JSON does not in itself create an + arbitrary code execution vulnerability. .. seealso:: The :mod:`json` module: a standard library module allowing JSON @@ -437,7 +451,7 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, how they can be loaded, potentially reducing security risks. Refer to :ref:`pickle-restrict` for details. - .. audit-event:: pickle.find_class "module name" + .. audit-event:: pickle.find_class module,name pickle.Unpickler.find_class .. class:: PickleBuffer(buffer) diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index d72b660d6ea9f6..28b42fa60c1859 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -39,6 +39,14 @@ The :mod:`poplib` module provides two classes: connection attempt (if not specified, the global default timeout setting will be used). + .. audit-event:: poplib.connect self,host,port poplib.POP3 + + .. audit-event:: poplib.putline self,line poplib.POP3 + + All commands will raise an :ref:`auditing event ` + ``poplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. + .. class:: POP3_SSL(host, port=POP3_SSL_PORT, keyfile=None, certfile=None, timeout=None, context=None) @@ -54,6 +62,14 @@ The :mod:`poplib` module provides two classes: point to PEM-formatted private key and certificate chain files, respectively, for the SSL connection. + .. audit-event:: poplib.connect self,host,port poplib.POP3_SSL + + .. audit-event:: poplib.putline self,line popplib.POP3_SSL + + All commands will raise an :ref:`auditing event ` + ``poplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. + .. versionchanged:: 3.2 *context* parameter added. diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst index 9cbc5505aee6f3..2105af9215f769 100644 --- a/Doc/library/posix.rst +++ b/Doc/library/posix.rst @@ -44,8 +44,8 @@ by defining the relevant size and offset types as 64-bit values. Such files are sometimes referred to as :dfn:`large files`. Large file support is enabled in Python when the size of an :c:type:`off_t` is -larger than a :c:type:`long` and the :c:type:`long long` type is available and is -at least as large as an :c:type:`off_t`. +larger than a :c:type:`long` and the :c:type:`long long` is at least as large +as an :c:type:`off_t`. It may be necessary to configure and compile Python with certain compiler flags to enable this mode. For example, it is enabled by default with recent versions of Irix, but with Solaris 2.6 and 2.7 you need to do something like:: diff --git a/Doc/library/random.rst b/Doc/library/random.rst index fcedba4dbc2052..90b86248e6e1c3 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -160,7 +160,8 @@ Functions for sequences The *weights* or *cum_weights* can use any numeric type that interoperates with the :class:`float` values returned by :func:`random` (that includes - integers, floats, and fractions but excludes decimals). + integers, floats, and fractions but excludes decimals). Weights are + assumed to be non-negative. For a given seed, the :func:`choices` function with equal weighting typically produces a different sequence than repeated calls to diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 5ef72b535ce8d4..158248c3d14794 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -942,7 +942,7 @@ form. >>> print('|'.join(map(re.escape, sorted(operators, reverse=True)))) /|\-|\+|\*\*|\* - This functions must not be used for the replacement string in :func:`sub` + This function must not be used for the replacement string in :func:`sub` and :func:`subn`, only backslashes should be escaped. For example:: >>> digits_re = r'\d+' diff --git a/Doc/library/select.rst b/Doc/library/select.rst index 733a91e20b684d..8f5a2cea9257cc 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -480,13 +480,14 @@ Kqueue Objects Create a kqueue object from a given file descriptor. -.. method:: kqueue.control(changelist, max_events[, timeout=None]) -> eventlist +.. method:: kqueue.control(changelist, max_events[, timeout]) -> eventlist Low level interface to kevent - - changelist must be an iterable of kevent object or ``None`` + - changelist must be an iterable of kevent objects or ``None`` - max_events must be 0 or a positive integer - - timeout in seconds (floats possible) + - timeout in seconds (floats possible); the default is ``None``, + to wait forever .. versionchanged:: 3.5 The function is now retried with a recomputed timeout when interrupted by diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index dcb2a16cff98cb..a51e068ebd593a 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -249,6 +249,8 @@ Directory and files operations   as arguments. By default, :func:`~shutil.copy2` is used, but any function   that supports the same signature (like :func:`~shutil.copy`) can be used. + .. audit-event:: shutil.copytree src,dst shutil.copytree + .. versionchanged:: 3.3 Copy metadata when *symlinks* is false. Now returns *dst*. @@ -296,10 +298,16 @@ Directory and files operations *excinfo*, will be the exception information returned by :func:`sys.exc_info`. Exceptions raised by *onerror* will not be caught. + .. audit-event:: shutil.rmtree path shutil.rmtree + .. versionchanged:: 3.3 Added a symlink attack resistant version that is used automatically if platform supports fd-based functions. + .. versionchanged:: 3.8 + On Windows, will no longer delete the contents of a directory junction + before removing the junction. + .. attribute:: rmtree.avoids_symlink_attacks Indicates whether the current platform and implementation provides a @@ -558,6 +566,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. The *verbose* argument is unused and deprecated. + .. audit-event:: shutil.make_archive base_name,format,root_dir,base_dir shutil.make_archive + .. versionchanged:: 3.8 The modern pax (POSIX.1-2001) format is now used instead of the legacy GNU format for archives created with ``format="tar"``. diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 01200b4df8803e..8fecc2b7eed0e0 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -95,7 +95,7 @@ The variables defined in the :mod:`signal` module are: All the signal numbers are defined symbolically. For example, the hangup signal is defined as :const:`signal.SIGHUP`; the variable names are identical to the - names used in C programs, as found in ````. The Unix man page for + names used in C programs, as found in ````. The Unix man page for ':c:func:`signal`' lists the existing signals (on some systems this is :manpage:`signal(2)`, on others the list is in :manpage:`signal(7)`). Note that not all systems define the same set of signal names; only those names defined by @@ -193,10 +193,10 @@ The :mod:`signal` module defines the following functions: canceled (only one alarm can be scheduled at any time). The returned value is then the number of seconds before any previously set alarm was to have been delivered. If *time* is zero, no alarm is scheduled, and any scheduled alarm is - canceled. If the return value is zero, no alarm is currently scheduled. (See - the Unix man page :manpage:`alarm(2)`.) + canceled. If the return value is zero, no alarm is currently scheduled. - .. availability:: Unix. + .. availability:: Unix. See the man page :manpage:`alarm(2)` for further + information. .. function:: getsignal(signalnum) @@ -231,8 +231,10 @@ The :mod:`signal` module defines the following functions: .. function:: pause() Cause the process to sleep until a signal is received; the appropriate handler - will then be called. Returns nothing. Not on Windows. (See the Unix man page - :manpage:`signal(2)`.) + will then be called. Returns nothing. + + .. availability:: Unix. See the man page :manpage:`signal(2)` for further + information. See also :func:`sigwait`, :func:`sigwaitinfo`, :func:`sigtimedwait` and :func:`sigpending`. @@ -262,8 +264,8 @@ The :mod:`signal` module defines the following functions: If *signalnum* is 0, then no signal is sent, but error checking is still performed; this can be used to check if the target thread is still running. - .. availability:: Unix (see the man page :manpage:`pthread_kill(3)` for further - information). + .. availability:: Unix. See the man page :manpage:`pthread_kill(3)` for further + information. See also :func:`os.kill`. @@ -293,7 +295,7 @@ The :mod:`signal` module defines the following functions: For example, ``signal.pthread_sigmask(signal.SIG_BLOCK, [])`` reads the signal mask of the calling thread. - .. availability:: Unix. See the man page :manpage:`sigprocmask(3)` and + .. availability:: Unix. See the man page :manpage:`sigprocmask(3)` and :manpage:`pthread_sigmask(3)` for further information. See also :func:`pause`, :func:`sigpending` and :func:`sigwait`. @@ -380,8 +382,8 @@ The :mod:`signal` module defines the following functions: calls will be restarted when interrupted by signal *signalnum*, otherwise system calls will be interrupted. Returns nothing. - .. availability:: Unix (see the man page :manpage:`siginterrupt(3)` - for further information). + .. availability:: Unix. See the man page :manpage:`siginterrupt(3)` + for further information. Note that installing a signal handler with :func:`signal` will reset the restart behaviour to interruptible by implicitly calling @@ -394,7 +396,7 @@ The :mod:`signal` module defines the following functions: be a callable Python object taking two arguments (see below), or one of the special values :const:`signal.SIG_IGN` or :const:`signal.SIG_DFL`. The previous signal handler will be returned (see the description of :func:`getsignal` - above). (See the Unix man page :manpage:`signal(2)`.) + above). (See the Unix man page :manpage:`signal(2)` for further information.) When threads are enabled, this function can only be called from the main thread; attempting to call it from other threads will cause a :exc:`ValueError` @@ -420,8 +422,8 @@ The :mod:`signal` module defines the following functions: thread (i.e., the signals which have been raised while blocked). Return the set of the pending signals. - .. availability:: Unix (see the man page :manpage:`sigpending(2)` for further - information). + .. availability:: Unix. See the man page :manpage:`sigpending(2)` for further + information. See also :func:`pause`, :func:`pthread_sigmask` and :func:`sigwait`. @@ -434,8 +436,8 @@ The :mod:`signal` module defines the following functions: signals specified in the signal set *sigset*. The function accepts the signal (removes it from the pending list of signals), and returns the signal number. - .. availability:: Unix (see the man page :manpage:`sigwait(3)` for further - information). + .. availability:: Unix. See the man page :manpage:`sigwait(3)` for further + information. See also :func:`pause`, :func:`pthread_sigmask`, :func:`sigpending`, :func:`sigwaitinfo` and :func:`sigtimedwait`. @@ -459,8 +461,8 @@ The :mod:`signal` module defines the following functions: :attr:`si_errno`, :attr:`si_pid`, :attr:`si_uid`, :attr:`si_status`, :attr:`si_band`. - .. availability:: Unix (see the man page :manpage:`sigwaitinfo(2)` for further - information). + .. availability:: Unix. See the man page :manpage:`sigwaitinfo(2)` for further + information. See also :func:`pause`, :func:`sigwait` and :func:`sigtimedwait`. @@ -478,8 +480,8 @@ The :mod:`signal` module defines the following functions: specifying a timeout. If *timeout* is specified as :const:`0`, a poll is performed. Returns :const:`None` if a timeout occurs. - .. availability:: Unix (see the man page :manpage:`sigtimedwait(2)` for further - information). + .. availability:: Unix. See the man page :manpage:`sigtimedwait(2)` for further + information. See also :func:`pause`, :func:`sigwait` and :func:`sigwaitinfo`. diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 2c3a5f0c6f7296..6176c35a0e4a50 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -55,6 +55,12 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). (250, b'Ok') >>> + .. audit-event:: smtplib.send self,data smtplib.SMTP + + All commands will raise an :ref:`auditing event ` + ``smtplib.SMTP.send`` with arguments ``self`` and ``data``, + where ``data`` is the bytes about to be sent to the remote host. + .. versionchanged:: 3.3 Support for the :keyword:`with` statement was added. @@ -242,6 +248,8 @@ An :class:`SMTP` instance has the following methods: 2-tuple of the response code and message sent by the server in its connection response. + .. audit-event:: smtplib.connect self,host,port smtplib.SMTP.connect + .. method:: SMTP.helo(name='') diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index e0dbbb4c0d3201..12416e0f4a9fd9 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -374,6 +374,9 @@ Constants .. availability:: Linux >= 2.6.25. + .. note:: + The :data:`CAN_BCM_CAN_FD_FRAME` flag is only available on Linux >= 4.8. + .. versionadded:: 3.4 .. data:: CAN_RAW_FD_FRAMES @@ -526,7 +529,7 @@ The following functions all create :ref:`socket objects `. The newly created socket is :ref:`non-inheritable `. - .. audit-event:: socket.__new__ "self family type protocol" + .. audit-event:: socket.__new__ self,family,type,protocol socket.socket .. versionchanged:: 3.3 The AF_CAN family was added. @@ -720,7 +723,7 @@ The :mod:`socket` module also offers various network-related services: :const:`AF_INET6`), and is meant to be passed to the :meth:`socket.connect` method. - .. audit-event:: socket.getaddrinfo "host port family type protocol" + .. audit-event:: socket.getaddrinfo host,port,family,type,protocol socket.getaddrinfo The following example fetches address information for a hypothetical TCP connection to ``example.org`` on port 80 (results may differ on your @@ -757,7 +760,7 @@ The :mod:`socket` module also offers various network-related services: interface. :func:`gethostbyname` does not support IPv6 name resolution, and :func:`getaddrinfo` should be used instead for IPv4/v6 dual stack support. - .. audit-event:: socket.gethostbyname hostname + .. audit-event:: socket.gethostbyname hostname socket.gethostbyname .. function:: gethostbyname_ex(hostname) @@ -771,7 +774,7 @@ The :mod:`socket` module also offers various network-related services: resolution, and :func:`getaddrinfo` should be used instead for IPv4/v6 dual stack support. - .. audit-event:: socket.gethostbyname hostname + .. audit-event:: socket.gethostbyname hostname socket.gethostbyname_ex .. function:: gethostname() @@ -779,7 +782,7 @@ The :mod:`socket` module also offers various network-related services: Return a string containing the hostname of the machine where the Python interpreter is currently executing. - .. audit-event:: socket.gethostname + .. audit-event:: socket.gethostname "" socket.gethostname Note: :func:`gethostname` doesn't always return the fully qualified domain name; use :func:`getfqdn` for that. @@ -795,7 +798,7 @@ The :mod:`socket` module also offers various network-related services: domain name, use the function :func:`getfqdn`. :func:`gethostbyaddr` supports both IPv4 and IPv6. - .. audit-event:: socket.gethostbyaddr ip_address + .. audit-event:: socket.gethostbyaddr ip_address socket.gethostbyaddr .. function:: getnameinfo(sockaddr, flags) @@ -810,7 +813,7 @@ The :mod:`socket` module also offers various network-related services: For more information about *flags* you can consult :manpage:`getnameinfo(3)`. - .. audit-event:: socket.getnameinfo sockaddr + .. audit-event:: socket.getnameinfo sockaddr socket.getnameinfo .. function:: getprotobyname(protocolname) @@ -827,7 +830,7 @@ The :mod:`socket` module also offers various network-related services: service. The optional protocol name, if given, should be ``'tcp'`` or ``'udp'``, otherwise any protocol will match. - .. audit-event:: socket.getservbyname "servicename protocolname" + .. audit-event:: socket.getservbyname servicename,protocolname socket.getservbyname .. function:: getservbyport(port[, protocolname]) @@ -836,7 +839,7 @@ The :mod:`socket` module also offers various network-related services: service. The optional protocol name, if given, should be ``'tcp'`` or ``'udp'``, otherwise any protocol will match. - .. audit-event:: socket.getservbyport "port protocolname" + .. audit-event:: socket.getservbyport port,protocolname socket.getservbyport .. function:: ntohl(x) @@ -1021,7 +1024,7 @@ The :mod:`socket` module also offers various network-related services: Set the machine's hostname to *name*. This will raise an :exc:`OSError` if you don't have enough rights. - .. audit-event:: socket.sethostname name + .. audit-event:: socket.sethostname name socket.sethostname .. availability:: Unix. @@ -1107,7 +1110,7 @@ to sockets. Bind the socket to *address*. The socket must not already be bound. (The format of *address* depends on the address family --- see above.) - .. audit-event:: socket.bind "self address" + .. audit-event:: socket.bind self,address socket.socket.bind .. method:: socket.close() @@ -1145,7 +1148,7 @@ to sockets. :exc:`InterruptedError` exception if the connection is interrupted by a signal (or the exception raised by the signal handler). - .. audit-event:: socket.connect "self address" + .. audit-event:: socket.connect self,address socket.socket.connect .. versionchanged:: 3.5 The method now waits until the connection completes instead of raising an @@ -1163,7 +1166,7 @@ to sockets. :c:data:`errno` variable. This is useful to support, for example, asynchronous connects. - .. audit-event:: socket.connect "self address" + .. audit-event:: socket.connect self,address socket.socket.connect_ex .. method:: socket.detach() @@ -1505,7 +1508,7 @@ to sockets. bytes sent. (The format of *address* depends on the address family --- see above.) - .. audit-event:: socket.sendto "self address" + .. audit-event:: socket.sendto self,address socket.socket.sendto .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise @@ -1546,7 +1549,7 @@ to sockets. .. availability:: most Unix platforms, possibly others. - .. audit-event:: socket.sendmsg "self address" + .. audit-event:: socket.sendmsg self,address socket.socket.sendmsg .. versionadded:: 3.3 diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 20fca54aab144f..67ea2b1d776638 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -224,6 +224,8 @@ Module functions and constants More information about this feature, including a list of recognized options, can be found in the `SQLite URI documentation `_. + .. audit-event:: sqlite3.connect database sqlite3.connect + .. versionchanged:: 3.4 Added the *uri* parameter. diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index c8f6904f9b1f7a..f48a0a9faa6b0c 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -425,3 +425,13 @@ for more detail on the meaning of these constants. FILE_ATTRIBUTE_VIRTUAL .. versionadded:: 3.5 + +On Windows, the following constants are available for comparing against the +``st_reparse_tag`` member returned by :func:`os.lstat`. These are well-known +constants, but are not an exhaustive list. + +.. data:: IO_REPARSE_TAG_SYMLINK + IO_REPARSE_TAG_MOUNT_POINT + IO_REPARSE_TAG_APPEXECLINK + + .. versionadded:: 3.8 diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index bc841fda72f887..bdd706d0a93e3c 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -19,17 +19,21 @@ -------------- This module provides functions for calculating mathematical statistics of -numeric (:class:`Real`-valued) data. - -.. note:: - - Unless explicitly noted otherwise, these functions support :class:`int`, - :class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`. - Behaviour with other types (whether in the numeric tower or not) is - currently unsupported. Mixed types are also undefined and - implementation-dependent. If your input data consists of mixed types, - you may be able to use :func:`map` to ensure a consistent result, e.g. - ``map(float, input_data)``. +numeric (:class:`~numbers.Real`-valued) data. + +The module is not intended to be a competitor to third-party libraries such +as `NumPy `_, `SciPy `_, or +proprietary full-featured statistics packages aimed at professional +statisticians such as Minitab, SAS and Matlab. It is aimed at the level of +graphing and scientific calculators. + +Unless explicitly noted, these functions support :class:`int`, +:class:`float`, :class:`~decimal.Decimal` and :class:`~fractions.Fraction`. +Behaviour with other types (whether in the numeric tower or not) is +currently unsupported. Collections with a mix of types are also undefined +and implementation-dependent. If your input data consists of mixed types, +you may be able to use :func:`map` to ensure a consistent result, for +example: ``map(float, input_data)``. Averages and measures of central location ----------------------------------------- @@ -102,14 +106,12 @@ However, for reading convenience, most of the examples show sorted sequences. .. note:: The mean is strongly affected by outliers and is not a robust estimator - for central location: the mean is not necessarily a typical example of the - data points. For more robust, although less efficient, measures of - central location, see :func:`median` and :func:`mode`. (In this case, - "efficient" refers to statistical efficiency rather than computational - efficiency.) + for central location: the mean is not necessarily a typical example of + the data points. For more robust measures of central location, see + :func:`median` and :func:`mode`. The sample mean gives an unbiased estimate of the true population mean, - which means that, taken on average over all the possible samples, + so that when taken on average over all the possible samples, ``mean(sample)`` converges on the true mean of the entire population. If *data* represents the entire population rather than a sample, then ``mean(data)`` is equivalent to calculating the true population mean μ. @@ -120,9 +122,8 @@ However, for reading convenience, most of the examples show sorted sequences. Convert *data* to floats and compute the arithmetic mean. This runs faster than the :func:`mean` function and it always returns a - :class:`float`. The result is highly accurate but not as perfect as - :func:`mean`. If the input dataset is empty, raises a - :exc:`StatisticsError`. + :class:`float`. The *data* may be a sequence or iterator. If the input + dataset is empty, raises a :exc:`StatisticsError`. .. doctest:: @@ -136,15 +137,20 @@ However, for reading convenience, most of the examples show sorted sequences. Convert *data* to floats and compute the geometric mean. + The geometric mean indicates the central tendency or typical value of the + *data* using the product of the values (as opposed to the arithmetic mean + which uses their sum). + Raises a :exc:`StatisticsError` if the input dataset is empty, if it contains a zero, or if it contains a negative value. + The *data* may be a sequence or iterator. No special efforts are made to achieve exact results. (However, this may change in the future.) .. doctest:: - >>> round(geometric_mean([54, 24, 36]), 9) + >>> round(geometric_mean([54, 24, 36]), 1) 36.0 .. versionadded:: 3.8 @@ -161,8 +167,16 @@ However, for reading convenience, most of the examples show sorted sequences. will be equivalent to ``3/(1/a + 1/b + 1/c)``. The harmonic mean is a type of average, a measure of the central - location of the data. It is often appropriate when averaging quantities - which are rates or ratios, for example speeds. For example: + location of the data. It is often appropriate when averaging + rates or ratios, for example speeds. + + Suppose a car travels 10 km at 40 km/hr, then another 10 km at 60 km/hr. + What is the average speed? + + .. doctest:: + + >>> harmonic_mean([40, 60]) + 48.0 Suppose an investor purchases an equal value of shares in each of three companies, with P/E (price/earning) ratios of 2.5, 3 and 10. @@ -173,9 +187,6 @@ However, for reading convenience, most of the examples show sorted sequences. >>> harmonic_mean([2.5, 3, 10]) # For an equal investment portfolio. 3.6 - Using the arithmetic mean would give an average of about 5.167, which - is too high. - :exc:`StatisticsError` is raised if *data* is empty, or any element is less than zero. @@ -188,9 +199,9 @@ However, for reading convenience, most of the examples show sorted sequences. middle two" method. If *data* is empty, :exc:`StatisticsError` is raised. *data* can be a sequence or iterator. - The median is a robust measure of central location, and is less affected by - the presence of outliers in your data. When the number of data points is - odd, the middle data point is returned: + The median is a robust measure of central location and is less affected by + the presence of outliers. When the number of data points is odd, the + middle data point is returned: .. doctest:: @@ -208,13 +219,10 @@ However, for reading convenience, most of the examples show sorted sequences. This is suited for when your data is discrete, and you don't mind that the median may not be an actual data point. - If your data is ordinal (supports order operations) but not numeric (doesn't - support addition), you should use :func:`median_low` or :func:`median_high` + If the data is ordinal (supports order operations) but not numeric (doesn't + support addition), consider using :func:`median_low` or :func:`median_high` instead. - .. seealso:: :func:`median_low`, :func:`median_high`, :func:`median_grouped` - - .. function:: median_low(data) Return the low median of numeric data. If *data* is empty, @@ -312,10 +320,12 @@ However, for reading convenience, most of the examples show sorted sequences. The mode (when it exists) is the most typical value and serves as a measure of central location. - If there are multiple modes, returns the first one encountered in the *data*. - If *data* is empty, :exc:`StatisticsError` is raised. + If there are multiple modes with the same frequency, returns the first one + encountered in the *data*. If the smallest or largest of those is + desired instead, use ``min(multimode(data))`` or ``max(multimode(data))``. + If the input *data* is empty, :exc:`StatisticsError` is raised. - ``mode`` assumes discrete data, and returns a single value. This is the + ``mode`` assumes discrete data and returns a single value. This is the standard treatment of the mode as commonly taught in schools: .. doctest:: @@ -323,8 +333,8 @@ However, for reading convenience, most of the examples show sorted sequences. >>> mode([1, 1, 2, 3, 3, 3, 3, 4]) 3 - The mode is unique in that it is the only statistic which also applies - to nominal (non-numeric) data: + The mode is unique in that it is the only statistic in this package that + also applies to nominal (non-numeric) data: .. doctest:: @@ -366,15 +376,16 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: pvariance(data, mu=None) - Return the population variance of *data*, a non-empty iterable of real-valued - numbers. Variance, or second moment about the mean, is a measure of the - variability (spread or dispersion) of data. A large variance indicates that - the data is spread out; a small variance indicates it is clustered closely - around the mean. + Return the population variance of *data*, a non-empty sequence or iterator + of real-valued numbers. Variance, or second moment about the mean, is a + measure of the variability (spread or dispersion) of data. A large + variance indicates that the data is spread out; a small variance indicates + it is clustered closely around the mean. - If the optional second argument *mu* is given, it should be the mean of - *data*. If it is missing or ``None`` (the default), the mean is - automatically calculated. + If the optional second argument *mu* is given, it is typically the mean of + the *data*. It can also be used to compute the second moment around a + point that is not the mean. If it is missing or ``None`` (the default), + the arithmetic mean is automatically calculated. Use this function to calculate the variance from the entire population. To estimate the variance from a sample, the :func:`variance` function is usually @@ -399,10 +410,6 @@ However, for reading convenience, most of the examples show sorted sequences. >>> pvariance(data, mu) 1.25 - This function does not attempt to verify that you have passed the actual mean - as *mu*. Using arbitrary values for *mu* may lead to invalid or impossible - results. - Decimals and Fractions are supported: .. doctest:: @@ -421,11 +428,11 @@ However, for reading convenience, most of the examples show sorted sequences. σ². When called on a sample instead, this is the biased sample variance s², also known as variance with N degrees of freedom. - If you somehow know the true population mean μ, you may use this function - to calculate the variance of a sample, giving the known population mean as - the second argument. Provided the data points are representative - (e.g. independent and identically distributed), the result will be an - unbiased estimate of the population variance. + If you somehow know the true population mean μ, you may use this + function to calculate the variance of a sample, giving the known + population mean as the second argument. Provided the data points are a + random sample of the population, the result will be an unbiased estimate + of the population variance. .. function:: stdev(data, xbar=None) @@ -500,44 +507,44 @@ However, for reading convenience, most of the examples show sorted sequences. :func:`pvariance` function as the *mu* parameter to get the variance of a sample. -.. function:: quantiles(dist, *, n=4, method='exclusive') +.. function:: quantiles(data, *, n=4, method='exclusive') - Divide *dist* into *n* continuous intervals with equal probability. + Divide *data* into *n* continuous intervals with equal probability. Returns a list of ``n - 1`` cut points separating the intervals. Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. Set *n* to 100 for percentiles which gives the 99 cuts points that separate - *dist* in to 100 equal sized groups. Raises :exc:`StatisticsError` if *n* + *data* into 100 equal sized groups. Raises :exc:`StatisticsError` if *n* is not least 1. - The *dist* can be any iterable containing sample data or it can be an - instance of a class that defines an :meth:`~inv_cdf` method. For meaningful - results, the number of data points in *dist* should be larger than *n*. + The *data* can be any iterable containing sample data. For meaningful + results, the number of data points in *data* should be larger than *n*. Raises :exc:`StatisticsError` if there are not at least two data points. - For sample data, the cut points are linearly interpolated from the + The cut points are linearly interpolated from the two nearest data points. For example, if a cut point falls one-third of the distance between two sample values, ``100`` and ``112``, the cut-point will evaluate to ``104``. The *method* for computing quantiles can be varied depending on - whether the data in *dist* includes or excludes the lowest and + whether the *data* includes or excludes the lowest and highest possible values from the population. The default *method* is "exclusive" and is used for data sampled from a population that can have more extreme values than found in the samples. The portion of the population falling below the *i-th* of - *m* data points is computed as ``i / (m + 1)``. + *m* sorted data points is computed as ``i / (m + 1)``. Given nine + sample values, the method sorts them and assigns the following + percentiles: 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%. Setting the *method* to "inclusive" is used for describing population - data or for samples that include the extreme points. The minimum - value in *dist* is treated as the 0th percentile and the maximum - value is treated as the 100th percentile. The portion of the - population falling below the *i-th* of *m* data points is computed as - ``(i - 1) / (m - 1)``. - - If *dist* is an instance of a class that defines an - :meth:`~inv_cdf` method, setting *method* has no effect. + data or for samples that are known to include the most extreme values + from the population. The minimum value in *data* is treated as the 0th + percentile and the maximum value is treated as the 100th percentile. + The portion of the population falling below the *i-th* of *m* sorted + data points is computed as ``(i - 1) / (m - 1)``. Given 11 sample + values, the method sorts them and assigns the following percentiles: + 0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%. .. doctest:: @@ -550,11 +557,6 @@ However, for reading convenience, most of the examples show sorted sequences. >>> [round(q, 1) for q in quantiles(data, n=10)] [81.0, 86.2, 89.0, 99.4, 102.5, 103.6, 106.0, 109.8, 111.0] - >>> # Quartile cut points for the standard normal distibution - >>> Z = NormalDist() - >>> [round(q, 4) for q in quantiles(Z, n=4)] - [-0.6745, 0.0, 0.6745] - .. versionadded:: 3.8 @@ -574,7 +576,7 @@ A single exception is defined: :class:`NormalDist` is a tool for creating and manipulating normal distributions of a `random variable `_. It is a -composite class that treats the mean and standard deviation of data +class that treats the mean and standard deviation of data measurements as a single entity. Normal distributions arise from the `Central Limit Theorem @@ -596,6 +598,18 @@ of applications in statistics. `_ of a normal distribution. + .. attribute:: median + + A read-only property for the `median + `_ of a normal + distribution. + + .. attribute:: mode + + A read-only property for the `mode + `_ of a normal + distribution. + .. attribute:: stdev A read-only property for the `standard deviation @@ -610,13 +624,14 @@ of applications in statistics. .. classmethod:: NormalDist.from_samples(data) - Makes a normal distribution instance computed from sample data. The - *data* can be any :term:`iterable` and should consist of values that - can be converted to type :class:`float`. + Makes a normal distribution instance with *mu* and *sigma* parameters + estimated from the *data* using :func:`fmean` and :func:`stdev`. - If *data* does not contain at least two elements, raises - :exc:`StatisticsError` because it takes at least one point to estimate - a central value and at least two points to estimate dispersion. + The *data* can be any :term:`iterable` and should consist of values + that can be converted to type :class:`float`. If *data* does not + contain at least two elements, raises :exc:`StatisticsError` because it + takes at least one point to estimate a central value and at least two + points to estimate dispersion. .. method:: NormalDist.samples(n, *, seed=None) @@ -630,10 +645,10 @@ of applications in statistics. .. method:: NormalDist.pdf(x) Using a `probability density function (pdf) - `_, - compute the relative likelihood that a random variable *X* will be near - the given value *x*. Mathematically, it is the ratio ``P(x <= X < - x+dx) / dx``. + `_, compute + the relative likelihood that a random variable *X* will be near the + given value *x*. Mathematically, it is the limit of the ratio ``P(x <= + X < x+dx) / dx`` as *dx* approaches zero. The relative likelihood is computed as the probability of a sample occurring in a narrow range divided by the width of the range (hence @@ -661,13 +676,21 @@ of applications in statistics. .. method:: NormalDist.overlap(other) - Compute the `overlapping coefficient (OVL) - `_ - between two normal distributions, giving a measure of agreement. + Measures the agreement between two normal probability distributions. Returns a value between 0.0 and 1.0 giving `the overlapping area for the two probability density functions `_. + .. method:: NormalDist.quantiles() + + Divide the normal distribution into *n* continuous intervals with + equal probability. Returns a list of (n - 1) cut points separating + the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate the normal distribution into 100 equal sized groups. + Instances of :class:`NormalDist` support addition, subtraction, multiplication and division by a constant. These operations are used for translation and scaling. For example: @@ -723,21 +746,11 @@ Find the `quartiles `_ and `deciles .. doctest:: - >>> [round(sat.inv_cdf(p)) for p in (0.25, 0.50, 0.75)] + >>> list(map(round, sat.quantiles())) [928, 1060, 1192] - >>> [round(sat.inv_cdf(p / 10)) for p in range(1, 10)] + >>> list(map(round, sat.quantiles(n=10))) [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310] -What percentage of men and women will have the same height in `two normally -distributed populations with known means and standard deviations -`_? - - >>> men = NormalDist(70, 4) - >>> women = NormalDist(65, 3.5) - >>> ovl = men.overlap(women) - >>> round(ovl * 100.0, 1) - 50.3 - To estimate the distribution for a model than isn't easy to solve analytically, :class:`NormalDist` can generate input samples for a `Monte Carlo simulation `_: @@ -748,11 +761,11 @@ Carlo simulation `_: ... return (3*x + 7*x*y - 5*y) / (11 * z) ... >>> n = 100_000 - >>> X = NormalDist(10, 2.5).samples(n) - >>> Y = NormalDist(15, 1.75).samples(n) - >>> Z = NormalDist(5, 1.25).samples(n) - >>> NormalDist.from_samples(map(model, X, Y, Z)) # doctest: +SKIP - NormalDist(mu=19.640137307085507, sigma=47.03273142191088) + >>> X = NormalDist(10, 2.5).samples(n, seed=3652260728) + >>> Y = NormalDist(15, 1.75).samples(n, seed=4582495471) + >>> Z = NormalDist(50, 1.25).samples(n, seed=6582483453) + >>> quantiles(map(model, X, Y, Z)) # doctest: +SKIP + [1.4591308524824727, 1.8035946855390597, 2.175091447274739] Normal distributions commonly arise in machine learning problems. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index fcb0da74e158b0..0f7c369ea51f9a 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1756,9 +1756,13 @@ expression support in the :mod:`re` module). .. method:: str.isspace() Return true if there are only whitespace characters in the string and there is - at least one character, false otherwise. Whitespace characters are those - characters defined in the Unicode character database as "Other" or "Separator" - and those with bidirectional property being one of "WS", "B", or "S". + at least one character, false otherwise. + + A character is *whitespace* if in the Unicode character database + (see :mod:`unicodedata`), either its general category is ``Zs`` + ("Separator, space"), or its bidirectional class is one of ``WS``, + ``B``, or ``S``. + .. method:: str.istitle() @@ -2736,8 +2740,8 @@ arbitrary binary data. The prefix(es) to search for may be any :term:`bytes-like object`. -.. method:: bytes.translate(table, delete=b'') - bytearray.translate(table, delete=b'') +.. method:: bytes.translate(table, /, delete=b'') + bytearray.translate(table, /, delete=b'') Return a copy of the bytes or bytearray object where all bytes occurring in the optional argument *delete* are removed, and the remaining bytes have @@ -3801,7 +3805,7 @@ copying. >>> z.nbytes 48 - Cast 1D/unsigned char to 2D/unsigned long:: + Cast 1D/unsigned long to 2D/unsigned long:: >>> buf = struct.pack("L"*6, *list(range(6))) >>> x = memoryview(buf) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 288dde6b3fe4c0..af8b9b358cc388 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -146,7 +146,7 @@ implementation as the built-in :meth:`~str.format` method. keyword arguments. For compound field names, these functions are only called for the first - component of the field name; Subsequent components are handled through + component of the field name; subsequent components are handled through normal attribute and indexing operations. So for example, the field expression '0.name' would cause diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 1a0fd73c6758c7..56b0a3614df533 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -197,44 +197,44 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``c`` | :c:type:`char` | bytes of length 1 | 1 | | +--------+--------------------------+--------------------+----------------+------------+ -| ``b`` | :c:type:`signed char` | integer | 1 | \(1),\(3) | +| ``b`` | :c:type:`signed char` | integer | 1 | \(1), \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``B`` | :c:type:`unsigned char` | integer | 1 | \(3) | +| ``B`` | :c:type:`unsigned char` | integer | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ | ``?`` | :c:type:`_Bool` | bool | 1 | \(1) | +--------+--------------------------+--------------------+----------------+------------+ -| ``h`` | :c:type:`short` | integer | 2 | \(3) | +| ``h`` | :c:type:`short` | integer | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``H`` | :c:type:`unsigned short` | integer | 2 | \(3) | +| ``H`` | :c:type:`unsigned short` | integer | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``i`` | :c:type:`int` | integer | 4 | \(3) | +| ``i`` | :c:type:`int` | integer | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``I`` | :c:type:`unsigned int` | integer | 4 | \(3) | +| ``I`` | :c:type:`unsigned int` | integer | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``l`` | :c:type:`long` | integer | 4 | \(3) | +| ``l`` | :c:type:`long` | integer | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``L`` | :c:type:`unsigned long` | integer | 4 | \(3) | +| ``L`` | :c:type:`unsigned long` | integer | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``q`` | :c:type:`long long` | integer | 8 | \(2), \(3) | +| ``q`` | :c:type:`long long` | integer | 8 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``Q`` | :c:type:`unsigned long | integer | 8 | \(2), \(3) | +| ``Q`` | :c:type:`unsigned long | integer | 8 | \(2) | | | long` | | | | +--------+--------------------------+--------------------+----------------+------------+ -| ``n`` | :c:type:`ssize_t` | integer | | \(4) | +| ``n`` | :c:type:`ssize_t` | integer | | \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``N`` | :c:type:`size_t` | integer | | \(4) | +| ``N`` | :c:type:`size_t` | integer | | \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``e`` | \(7) | float | 2 | \(5) | +| ``e`` | \(6) | float | 2 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ -| ``f`` | :c:type:`float` | float | 4 | \(5) | +| ``f`` | :c:type:`float` | float | 4 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ -| ``d`` | :c:type:`double` | float | 8 | \(5) | +| ``d`` | :c:type:`double` | float | 8 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ | ``s`` | :c:type:`char[]` | bytes | | | +--------+--------------------------+--------------------+----------------+------------+ | ``p`` | :c:type:`char[]` | bytes | | | +--------+--------------------------+--------------------+----------------+------------+ -| ``P`` | :c:type:`void \*` | integer | | \(6) | +| ``P`` | :c:type:`void \*` | integer | | \(5) | +--------+--------------------------+--------------------+----------------+------------+ .. versionchanged:: 3.3 @@ -254,11 +254,6 @@ Notes: standard mode, it is always represented by one byte. (2) - The ``'q'`` and ``'Q'`` conversion codes are available in native mode only if - the platform C compiler supports C :c:type:`long long`, or, on Windows, - :c:type:`__int64`. They are always available in standard modes. - -(3) When attempting to pack a non-integer using any of the integer conversion codes, if the non-integer has a :meth:`__index__` method then that method is called to convert the argument to an integer before packing. @@ -266,26 +261,26 @@ Notes: .. versionchanged:: 3.2 Use of the :meth:`__index__` method for non-integers is new in 3.2. -(4) +(3) The ``'n'`` and ``'N'`` conversion codes are only available for the native size (selected as the default or with the ``'@'`` byte order character). For the standard size, you can use whichever of the other integer formats fits your application. -(5) +(4) For the ``'f'``, ``'d'`` and ``'e'`` conversion codes, the packed representation uses the IEEE 754 binary32, binary64 or binary16 format (for ``'f'``, ``'d'`` or ``'e'`` respectively), regardless of the floating-point format used by the platform. -(6) +(5) The ``'P'`` format character is only available for the native byte ordering (selected as the default or with the ``'@'`` byte order character). The byte order character ``'='`` chooses to use little- or big-endian ordering based on the host system. The struct module does not interpret this as native ordering, so the ``'P'`` format is not available. -(7) +(6) The IEEE 754 binary16 "half precision" type was introduced in the 2008 revision of the `IEEE 754 standard `_. It has a sign bit, a 5-bit exponent and 11-bit precision (with 10 bits explicitly stored), diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index ede5c3c5a369e3..167ed9a6ead497 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -483,6 +483,13 @@ functions. The *start_new_session* parameter can take the place of a previously common use of *preexec_fn* to call os.setsid() in the child. + .. versionchanged:: 3.8 + + The *preexec_fn* parameter is no longer supported in subinterpreters. + The use of the parameter in a subinterpreter raises + :exc:`RuntimeError`. The new restriction may affect applications that + are deployed in mod_wsgi, uWSGI, and other embedded environments. + If *close_fds* is true, all file descriptors except :const:`0`, :const:`1` and :const:`2` will be closed before the child process is executed. Otherwise when *close_fds* is false, file descriptors obey their inheritable flag @@ -585,6 +592,13 @@ functions. with Popen(["ifconfig"], stdout=PIPE) as proc: log.write(proc.stdout.read()) + .. audit-event:: subprocess.Popen executable,args,cwd,env subprocess.Popen + + Popen and the other functions in this module that use it raise an + :ref:`auditing event ` ``subprocess.Popen`` with arguments + ``executable``, ``args``, ``cwd``, ``env``. The value for ``args`` + may be a single string or a list of strings, depending on platform. + .. versionchanged:: 3.2 Added context manager support. @@ -1159,12 +1173,12 @@ In the following examples, we assume that the relevant functions have already been imported from the :mod:`subprocess` module. -Replacing /bin/sh shell backquote -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Replacing :program:`/bin/sh` shell command substitution +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: bash - output=`mycmd myarg` + output=$(mycmd myarg) becomes:: @@ -1175,7 +1189,7 @@ Replacing shell pipeline .. code-block:: bash - output=`dmesg | grep hda` + output=$(dmesg | grep hda) becomes:: @@ -1192,7 +1206,7 @@ be used directly: .. code-block:: bash - output=`dmesg | grep hda` + output=$(dmesg | grep hda) becomes:: diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 5bde6870717c90..be1af371d428a5 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -16,6 +16,10 @@ always available. On POSIX systems where Python was built with the standard ``configure`` script, this contains the ABI flags as specified by :pep:`3149`. + .. versionchanged:: 3.8 + Default flags became an empty string (``m`` flag for pymalloc has been + removed). + .. versionadded:: 3.2 @@ -82,6 +86,9 @@ always available. The native equivalent of this function is :c:func:`PySys_Audit`. Using the native function is preferred when possible. + See the :ref:`audit events table ` for all events raised by + ``CPython``. + .. versionadded:: 3.8 @@ -162,7 +169,7 @@ always available. This function should be used for internal and specialized purposes only. - .. audit-event:: sys._current_frames + .. audit-event:: sys._current_frames "" sys._current_frames .. function:: breakpointhook() @@ -671,7 +678,7 @@ always available. that is deeper than the call stack, :exc:`ValueError` is raised. The default for *depth* is zero, returning the frame at the top of the call stack. - .. audit-event:: sys._getframe + .. audit-event:: sys._getframe "" sys._getframe .. impl-detail:: @@ -876,6 +883,10 @@ always available. .. versionadded:: 3.3 + .. note:: + + The addition of new required attributes must go through the normal PEP + process. See :pep:`421` for more information. .. data:: int_info @@ -906,6 +917,12 @@ always available. read, so that you can set this hook there. The :mod:`site` module :ref:`sets this `. + .. audit-event:: cpython.run_interactivehook hook sys.__interactivehook__ + + Raises an :ref:`auditing event ` + ``cpython.run_interactivehook`` with the hook object as the argument when + the hook is called on startup. + .. versionadded:: 3.4 @@ -1186,7 +1203,7 @@ always available. ``'return'``, ``'c_call'``, ``'c_return'``, or ``'c_exception'``. *arg* depends on the event type. - .. audit-event:: sys.setprofile + .. audit-event:: sys.setprofile "" sys.setprofile The events have the following meaning: @@ -1259,7 +1276,8 @@ always available. The trace function is invoked (with *event* set to ``'call'``) whenever a new local scope is entered; it should return a reference to a local trace - function to be used that scope, or ``None`` if the scope shouldn't be traced. + function to be used for the new scope, or ``None`` if the scope shouldn't be + traced. The local trace function should return a reference to itself (or to another function for further tracing in that scope), or ``None`` to turn off tracing @@ -1308,7 +1326,7 @@ always available. For more information on code and frame objects, refer to :ref:`types`. - .. audit-event:: sys.settrace + .. audit-event:: sys.settrace "" sys.settrace .. impl-detail:: @@ -1330,9 +1348,9 @@ always available. first time. The *finalizer* will be called when an asynchronous generator is about to be garbage collected. - .. audit-event:: sys.set_asyncgen_hooks_firstiter + .. audit-event:: sys.set_asyncgen_hooks_firstiter "" sys.set_asyncgen_hooks - .. audit-event:: sys.set_asyncgen_hooks_finalizer + .. audit-event:: sys.set_asyncgen_hooks_finalizer "" sys.set_asyncgen_hooks Two auditing events are raised because the underlying API consists of two calls, each of which must raise its own event. @@ -1514,13 +1532,21 @@ always available. * *err_msg*: Error message, can be ``None``. * *object*: Object causing the exception, can be ``None``. - :func:`sys.unraisablehook` can be overridden to control how unraisable - exceptions are handled. - The default hook formats *err_msg* and *object* as: ``f'{err_msg}: {object!r}'``; use "Exception ignored in" error message if *err_msg* is ``None``. + :func:`sys.unraisablehook` can be overridden to control how unraisable + exceptions are handled. + + Storing *exc_value* using a custom hook can create a reference cycle. It + should be cleared explicitly to break the reference cycle when the + exception is no longer needed. + + Storing *object* using a custom hook can resurrect it if it is set to an + object which is being finalized. Avoid storing *object* after the custom + hook completes to avoid resurrecting objects. + See also :func:`excepthook` which handles uncaught exceptions. .. versionadded:: 3.8 diff --git a/Doc/library/telnetlib.rst b/Doc/library/telnetlib.rst index 4ba426425277f9..48a9aea50dddd1 100644 --- a/Doc/library/telnetlib.rst +++ b/Doc/library/telnetlib.rst @@ -29,7 +29,7 @@ Character), EL (Erase Line), GA (Go Ahead), SB (Subnegotiation Begin). .. class:: Telnet(host=None, port=0[, timeout]) :class:`Telnet` represents a connection to a Telnet server. The instance is - initially not connected by default; the :meth:`open` method must be used to + initially not connected by default; the :meth:`~Telnet.open` method must be used to establish a connection. Alternatively, the host name and optional port number can be passed to the constructor too, in which case the connection to the server will be established before the constructor returns. The optional @@ -141,6 +141,8 @@ Telnet Objects Do not try to reopen an already connected instance. + .. audit-event:: telnetlib.Telnet.open self,host,port telnetlib.Telnet.open + .. method:: Telnet.msg(msg, *args) @@ -176,6 +178,8 @@ Telnet Objects block if the connection is blocked. May raise :exc:`OSError` if the connection is closed. + .. audit-event:: telnetlib.Telnet.write self,buffer telnetlib.Telnet.write + .. versionchanged:: 3.3 This method used to raise :exc:`socket.error`, which is now an alias of :exc:`OSError`. diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index daa6f621f1378e..0793e43df0145c 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -62,6 +62,8 @@ The module defines the following user-callable items: The :py:data:`os.O_TMPFILE` flag is used if it is available and works (Linux-specific, requires Linux kernel 3.11 or later). + .. audit-event:: tempfile.mkstemp fullpath tempfile.TemporaryFile + .. versionchanged:: 3.5 The :py:data:`os.O_TMPFILE` flag is now used if available. @@ -85,6 +87,8 @@ The module defines the following user-callable items: attribute is the underlying true file object. This file-like object can be used in a :keyword:`with` statement, just like a normal file. + .. audit-event:: tempfile.mkstemp fullpath tempfile.NamedTemporaryFile + .. versionchanged:: 3.8 Added *errors* parameter. @@ -130,6 +134,8 @@ The module defines the following user-callable items: The directory can be explicitly cleaned up by calling the :func:`cleanup` method. + .. audit-event:: tempfile.mkdtemp fullpath tempfile.TemporaryDirectory + .. versionadded:: 3.2 @@ -177,6 +183,8 @@ The module defines the following user-callable items: file (as would be returned by :func:`os.open`) and the absolute pathname of that file, in that order. + .. audit-event:: tempfile.mkstemp fullpath tempfile.mkstemp + .. versionchanged:: 3.5 *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to obtain a bytes return value. Prior to this, only str was allowed. @@ -198,6 +206,8 @@ The module defines the following user-callable items: :func:`mkdtemp` returns the absolute pathname of the new directory. + .. audit-event:: tempfile.mkdtemp fullpath tempfile.mkdtemp + .. versionchanged:: 3.5 *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to obtain a bytes return value. Prior to this, only str was allowed. diff --git a/Doc/library/test.rst b/Doc/library/test.rst index b7a2595708d0ce..da6a85d340be34 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -356,11 +356,28 @@ The :mod:`test.support` module defines the following constants: Check for presence of docstrings. + .. data:: TEST_HTTP_URL Define the URL of a dedicated HTTP server for the network tests. +.. data:: ALWAYS_EQ + + Object that is equal to anything. Used to test mixed type comparison. + + +.. data:: LARGEST + + Object that is greater than anything (except itself). + Used to test mixed type comparison. + + +.. data:: SMALLEST + + Object that is less than anything (except itself). + Used to test mixed type comparison. + The :mod:`test.support` module defines the following functions: @@ -1081,11 +1098,52 @@ The :mod:`test.support` module defines the following functions: :exc:`PermissionError` is raised. +.. function:: catch_threading_exception() + + Context manager catching :class:`threading.Thread` exception using + :func:`threading.excepthook`. + + Attributes set when an exception is catched: + + * ``exc_type`` + * ``exc_value`` + * ``exc_traceback`` + * ``thread`` + + See :func:`threading.excepthook` documentation. + + These attributes are deleted at the context manager exit. + + Usage:: + + with support.catch_threading_exception() as cm: + # code spawning a thread which raises an exception + ... + + # check the thread exception, use cm attributes: + # exc_type, exc_value, exc_traceback, thread + ... + + # exc_type, exc_value, exc_traceback, thread attributes of cm no longer + # exists at this point + # (to avoid reference cycles) + + .. versionadded:: 3.8 + + .. function:: catch_unraisable_exception() Context manager catching unraisable exception using :func:`sys.unraisablehook`. + Storing the exception value (``cm.unraisable.exc_value``) creates a + reference cycle. The reference cycle is broken explicitly when the context + manager exits. + + Storing the object (``cm.unraisable.object``) can resurrect it if it is set + to an object which is being finalized. Exiting the context manager clears + the stored object. + Usage:: with support.catch_unraisable_exception() as cm: diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index d254466c9a320d..0f11ef401569de 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -77,6 +77,9 @@ functions should be good enough; otherwise, you should use an instance of equal: the lines ``" hello"`` and ``"\thello"`` are considered to have no common leading whitespace. + Lines containing only whitespace are ignored in the input and normalized to a + single newline character in the output. + For example:: def test(): diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 7fb9ac9979e42f..eb9da2b9119745 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -58,6 +58,14 @@ This module defines the following functions: :func:`threading.excepthook` can be overridden to control how uncaught exceptions raised by :func:`Thread.run` are handled. + Storing *exc_value* using a custom hook can create a reference cycle. It + should be cleared explicitly to break the reference cycle when the + exception is no longer needed. + + Storing *object* using a custom hook can resurrect it if it is set to an + object which is being finalized. Avoid storing *object* after the custom + hook completes to avoid resurrecting objects. + .. seealso:: :func:`sys.excepthook` handles uncaught exceptions. @@ -82,7 +90,7 @@ This module defines the following functions: Its value may be used to uniquely identify this particular thread system-wide (until the thread terminates, after which the value may be recycled by the OS). - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD. + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. .. versionadded:: 3.8 @@ -355,7 +363,7 @@ since it is impossible to detect the termination of alien threads. system-wide) from the time the thread is created until the thread has been terminated. - .. availability:: Require :func:`get_native_id` function. + .. availability:: Requires :func:`get_native_id` function. .. versionadded:: 3.8 diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 6d0ceafa522ad6..4faa0bbebc5b1f 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -608,7 +608,7 @@ Functions .. function:: time_ns() -> int - Similar to :func:`time` but returns time as an integer number of nanoseconds + Similar to :func:`~time.time` but returns time as an integer number of nanoseconds since the epoch_. .. versionadded:: 3.7 diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 640fb2ec61d359..e1fc051d9597c6 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -31,7 +31,7 @@ installed, so you can read the Tcl/Tk documentation specific to that version. `TKDocs `_ Extensive tutorial plus friendlier widget pages for some of the widgets. - `Tkinter reference: a GUI for Python `_ + `Tkinter 8.5 reference: a GUI for Python `_ On-line reference material. `Tkinter docs from effbot `_ @@ -41,7 +41,7 @@ installed, so you can read the Tcl/Tk documentation specific to that version. Book by Mark Lutz, has excellent coverage of Tkinter. `Modern Tkinter for Busy Python Developers `_ - Book by Mark Rozerman about building attractive and modern graphical user interfaces with Python and Tkinter. + Book by Mark Roseman about building attractive and modern graphical user interfaces with Python and Tkinter. `Python and Tkinter Programming `_ Book by John Grayson (ISBN 1-884777-81-3). diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 1a766c29a57a53..12efde1316570f 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1001,6 +1001,13 @@ The module defines the following classes, functions and decorators: .. versionadded:: 3.8 +.. class:: ForwardRef + + A class used for internal typing representation of string forward references. + For example, ``List["SomeClass"]`` is implicitly transformed into + ``List[ForwardRef("SomeClass")]``. This class should not be instantiated by + a user, but may be used by introspection tools. + .. function:: NewType(typ) A helper function to indicate a distinct types to a typechecker, @@ -1238,7 +1245,8 @@ The module defines the following classes, functions and decorators: .. data:: Tuple Tuple type; ``Tuple[X, Y]`` is the type of a tuple of two items - with the first item of type X and the second of type Y. + with the first item of type X and the second of type Y. The type of + the empty tuple can be written as ``Tuple[()]``. Example: ``Tuple[T1, T2]`` is a tuple of two elements corresponding to type variables T1 and T2. ``Tuple[int, float, str]`` is a tuple diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 46e8ef38ab11bd..304ba532274ddb 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -286,7 +286,7 @@ the *new_callable* argument to :func:`patch`. used to set attributes on the mock after it is created. See the :meth:`configure_mock` method for details. - .. method:: assert_called(*args, **kwargs) + .. method:: assert_called() Assert that the mock was called at least once. @@ -297,7 +297,7 @@ the *new_callable* argument to :func:`patch`. .. versionadded:: 3.6 - .. method:: assert_called_once(*args, **kwargs) + .. method:: assert_called_once() Assert that the mock was called exactly once. @@ -317,8 +317,8 @@ the *new_callable* argument to :func:`patch`. .. method:: assert_called_with(*args, **kwargs) - This method is a convenient way of asserting that calls are made in a - particular way: + This method is a convenient way of asserting that the last call has been + made in a particular way: >>> mock = Mock() >>> mock.method(1, 2, 3, test='wow') diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index a53c969ec629a5..448bc67854704a 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -95,9 +95,9 @@ The :mod:`urllib.request` module defines the following functions: parameter to ``urllib.urlopen``, can be obtained by using :class:`ProxyHandler` objects. - .. audit-event:: urllib.Request "fullurl data headers method" + .. audit-event:: urllib.Request fullurl,data,headers,method urllib.request.urlopen - The default opener raises an :func:`auditing event ` + The default opener raises an :ref:`auditing event ` ``urllib.Request`` with arguments ``fullurl``, ``data``, ``headers``, ``method`` taken from the request object. @@ -227,7 +227,7 @@ The following classes are provided: is not None, ``Content-Type: application/x-www-form-urlencoded`` will be added as a default. - The final two arguments are only of interest for correct handling + The next two arguments are only of interest for correct handling of third-party HTTP cookies: *origin_req_host* should be the request-host of the origin diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 4f083a3181e7a9..1e825c3c21e843 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -47,7 +47,7 @@ Creating virtual environments A virtual environment is a directory tree which contains Python executable files and other files which indicate that it is a virtual environment. - Common installation tools such as ``Setuptools`` and ``pip`` work as + Common installation tools such as setuptools_ and pip_ work as expected with virtual environments. In other words, when a virtual environment is active, they install Python packages into the virtual environment without needing to be told to do so explicitly. @@ -64,24 +64,25 @@ Creating virtual environments Python installation). When a virtual environment is active, any options that change the - installation path will be ignored from all distutils configuration files to - prevent projects being inadvertently installed outside of the virtual - environment. + installation path will be ignored from all :mod:`distutils` configuration + files to prevent projects being inadvertently installed outside of the + virtual environment. When working in a command shell, users can make a virtual environment active by running an ``activate`` script in the virtual environment's executables - directory (the precise filename is shell-dependent), which prepends the - virtual environment's directory for executables to the ``PATH`` environment - variable for the running shell. There should be no need in other - circumstances to activate a virtual environment—scripts installed into - virtual environments have a "shebang" line which points to the virtual - environment's Python interpreter. This means that the script will run with - that interpreter regardless of the value of ``PATH``. On Windows, "shebang" - line processing is supported if you have the Python Launcher for Windows - installed (this was added to Python in 3.3 - see :pep:`397` for more - details). Thus, double-clicking an installed script in a Windows Explorer - window should run the script with the correct interpreter without there - needing to be any reference to its virtual environment in ``PATH``. + directory (the precise filename and command to use the file is + shell-dependent), which prepends the virtual environment's directory for + executables to the ``PATH`` environment variable for the running shell. There + should be no need in other circumstances to activate a virtual + environment; scripts installed into virtual environments have a "shebang" + line which points to the virtual environment's Python interpreter. This means + that the script will run with that interpreter regardless of the value of + ``PATH``. On Windows, "shebang" line processing is supported if you have the + Python Launcher for Windows installed (this was added to Python in 3.3 - see + :pep:`397` for more details). Thus, double-clicking an installed script in a + Windows Explorer window should run the script with the correct interpreter + without there needing to be any reference to its virtual environment in + ``PATH``. .. _venv-api: @@ -130,20 +131,20 @@ creation according to their needs, the :class:`EnvBuilder` class. Added the ``prompt`` parameter Creators of third-party virtual environment tools will be free to use the - provided ``EnvBuilder`` class as a base class. + provided :class:`EnvBuilder` class as a base class. The returned env-builder is an object which has a method, ``create``: .. method:: create(env_dir) - This method takes as required argument the path (absolute or relative to - the current directory) of the target directory which is to contain the + Create a virtual environment by specifying the target directory + (absolute or relative to the current directory) which is to contain the virtual environment. The ``create`` method will either create the environment in the specified directory, or raise an appropriate exception. - The ``create`` method of the ``EnvBuilder`` class illustrates the hooks - available for subclass customization:: + The ``create`` method of the :class:`EnvBuilder` class illustrates the + hooks available for subclass customization:: def create(self, env_dir): """ @@ -471,3 +472,7 @@ subclass which installs setuptools and pip into a created virtual environment:: This script is also available for download `online `_. + + +.. _setuptools: https://pypi.org/project/setuptools/ +.. _pip: https://pypi.org/project/pip/ diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index a28d71060f3836..b3c8e354553311 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -65,8 +65,8 @@ exposed by the :mod:`weakref` module for the benefit of advanced uses. Not all objects can be weakly referenced; those objects which can include class instances, functions written in Python (but not in C), instance methods, sets, -frozensets, some :term:`file objects `, :term:`generator`\s, type -objects, sockets, arrays, deques, regular expression pattern objects, and code +frozensets, some :term:`file objects `, :term:`generators `, +type objects, sockets, arrays, deques, regular expression pattern objects, and code objects. .. versionchanged:: 3.2 @@ -80,9 +80,10 @@ support weak references but can add support through subclassing:: obj = Dict(red=1, green=2, blue=3) # this object is weak referenceable -Other built-in types such as :class:`tuple` and :class:`int` do not support weak -references even when subclassed (This is an implementation detail and may be -different across various Python implementations.). +.. impl-detail:: + + Other built-in types such as :class:`tuple` and :class:`int` do not support weak + references even when subclassed. Extension types can easily be made to support weak references; see :ref:`weakref-support`. diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index 9dc5551398ee82..b7bfb655a71579 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -64,6 +64,8 @@ The following functions are defined: may work and start the operating system's associated program. However, this is neither supported nor portable. + .. audit-event:: webbrowser.open url webbrowser.open + .. function:: open_new(url) diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index e9c02610227331..cb67f2f39d7603 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -456,7 +456,7 @@ This module offers the following functions: *key* is an already open key, or one of the predefined :ref:`HKEY_* constants `. - Will generally raise :exc:`NotImplemented` if executed on a 32-bit operating + Will generally raise :exc:`NotImplementedError` if executed on a 32-bit operating system. If the key is not on the reflection list, the function succeeds but has no @@ -471,7 +471,7 @@ This module offers the following functions: *key* is an already open key, or one of the predefined :ref:`HKEY_* constants `. - Will generally raise :exc:`NotImplemented` if executed on a 32-bit operating + Will generally raise :exc:`NotImplementedError` if executed on a 32-bit operating system. Restoring reflection for a key does not affect reflection of any subkeys. @@ -486,7 +486,7 @@ This module offers the following functions: Returns ``True`` if reflection is disabled. - Will generally raise :exc:`NotImplemented` if executed on a 32-bit + Will generally raise :exc:`NotImplementedError` if executed on a 32-bit operating system. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index c4667315793e4c..9f46755c2685d1 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -659,7 +659,7 @@ Functions .. function:: tostring(element, encoding="us-ascii", method="xml", *, \ - xml_declaration=None, default_namespace=None, + xml_declaration=None, default_namespace=None, \ short_empty_elements=True) Generates a string representation of an XML element, including all @@ -677,9 +677,13 @@ Functions .. versionadded:: 3.8 The *xml_declaration* and *default_namespace* parameters. + .. versionchanged:: 3.8 + The :func:`tostring` function now preserves the attribute order + specified by the user. + .. function:: tostringlist(element, encoding="us-ascii", method="xml", *, \ - xml_declaration=None, default_namespace=None, + xml_declaration=None, default_namespace=None, \ short_empty_elements=True) Generates a string representation of an XML element, including all @@ -700,6 +704,10 @@ Functions .. versionadded:: 3.8 The *xml_declaration* and *default_namespace* parameters. + .. versionchanged:: 3.8 + The :func:`tostringlist` function now preserves the attribute order + specified by the user. + .. function:: XML(text, parser=None) @@ -930,6 +938,36 @@ Element Objects if element is None: print("element not found") + Prior to Python 3.8, the serialisation order of the XML attributes of + elements was artificially made predictable by sorting the attributes by + their name. Based on the now guaranteed ordering of dicts, this arbitrary + reordering was removed in Python 3.8 to preserve the order in which + attributes were originally parsed or created by user code. + + In general, user code should try not to depend on a specific ordering of + attributes, given that the `XML Information Set + `_ explicitly excludes the attribute + order from conveying information. Code should be prepared to deal with + any ordering on input. In cases where deterministic XML output is required, + e.g. for cryptographic signing or test data sets, canonical serialisation + is available with the :func:`canonicalize` function. + + In cases where canonical output is not applicable but a specific attribute + order is still desirable on output, code should aim for creating the + attributes directly in the desired order, to avoid perceptual mismatches + for readers of the code. In cases where this is difficult to achieve, a + recipe like the following can be applied prior to serialisation to enforce + an order independently from the Element creation:: + + def reorder_attributes(root): + for el in root.iter(): + attrib = el.attrib + if len(attrib) > 1: + # adjust attribute order, e.g. by sorting + attribs = sorted(attrib.items()) + attrib.clear() + attrib.update(attribs) + .. _elementtree-elementtree-objects: diff --git a/Doc/make.bat b/Doc/make.bat index e6604956ea916b..dfc622f66615df 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -117,13 +117,13 @@ if not exist "%BUILDDIR%" mkdir "%BUILDDIR%" rem PY_MISC_NEWS_DIR is also used by our Sphinx extension in tools/extensions/pyspecific.py if not defined PY_MISC_NEWS_DIR set PY_MISC_NEWS_DIR=%BUILDDIR%\%1 +if not exist "%PY_MISC_NEWS_DIR%" mkdir "%PY_MISC_NEWS_DIR%" if exist ..\Misc\NEWS ( echo.Copying Misc\NEWS to %PY_MISC_NEWS_DIR%\NEWS copy ..\Misc\NEWS "%PY_MISC_NEWS_DIR%\NEWS" > nul ) else if exist ..\Misc\NEWS.D ( if defined BLURB ( echo.Merging Misc/NEWS with %BLURB% - if not exist build mkdir build %BLURB% merge -f "%PY_MISC_NEWS_DIR%\NEWS" ) else ( echo.No Misc/NEWS file and Blurb is not available. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index fa47bf1c1619b5..8813f57587f01a 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1166,10 +1166,10 @@ Basic customization with appropriate arguments and then modifying the newly-created instance as necessary before returning it. - If :meth:`__new__` returns an instance of *cls*, then the new instance's - :meth:`__init__` method will be invoked like ``__init__(self[, ...])``, where - *self* is the new instance and the remaining arguments are the same as were - passed to :meth:`__new__`. + If :meth:`__new__` is invoked during object construction and it returns an + instance or subclass of *cls*, then the new instance’s :meth:`__init__` method + will be invoked like ``__init__(self[, ...])``, where *self* is the new instance + and the remaining arguments are the same as were passed to the object constructor. If :meth:`__new__` does not return an instance of *cls*, then the new instance's :meth:`__init__` method will not be invoked. @@ -1445,8 +1445,8 @@ Basic customization .. note:: - By default, the :meth:`__hash__` values of str, bytes and datetime - objects are "salted" with an unpredictable random value. Although they + By default, the :meth:`__hash__` values of str and bytes objects are + "salted" with an unpredictable random value. Although they remain constant within an individual Python process, they are not predictable between repeated invocations of Python. @@ -1618,21 +1618,32 @@ refers to the attribute whose name is the key of the property in the owner class' :attr:`~object.__dict__`. -.. method:: object.__get__(self, instance, owner) +.. method:: object.__get__(self, instance, owner=None) - Called to get the attribute of the owner class (class attribute access) or of an - instance of that class (instance attribute access). *owner* is always the owner - class, while *instance* is the instance that the attribute was accessed through, - or ``None`` when the attribute is accessed through the *owner*. This method - should return the (computed) attribute value or raise an :exc:`AttributeError` - exception. + Called to get the attribute of the owner class (class attribute access) or + of an instance of that class (instance attribute access). The optional + *owner* argument is the owner class, while *instance* is the instance that + the attribute was accessed through, or ``None`` when the attribute is + accessed through the *owner*. + This method should return the computed attribute value or raise an + :exc:`AttributeError` exception. + + :PEP:`252` specifies that :meth:`__get__` is callable with one or two + arguments. Python's own built-in descriptors support this specification; + however, it is likely that some third-party tools have descriptors + that require both arguments. Python's own :meth:`__getattribute__` + implementation always passes in both arguments whether they are required + or not. .. method:: object.__set__(self, instance, value) Called to set the attribute on an instance *instance* of the owner class to a new value, *value*. + Note, adding :meth:`__set__` or :meth:`__delete__` changes the kind of + descriptor to a "data descriptor". See :ref:`descriptor-invocation` for + more details. .. method:: object.__delete__(self, instance) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 8b7110615240ee..ee13c5f4c6fc23 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -337,6 +337,12 @@ all mutable objects.) Clashes between duplicate keys are not detected; the last datum (textually rightmost in the display) stored for a given key value prevails. +.. versionchanged:: 3.8 + Prior to Python 3.8, in dict comprehensions, the evaluation order of key + and value was not well-defined. In CPython, the value was evaluated before + the key. Starting with 3.8, the key is evaluated before the value, as + proposed by :pep:`572`. + .. _genexpr: @@ -1419,6 +1425,10 @@ built-in types. themselves. For example, if ``x = float('NaN')``, ``3 < x``, ``x < 3``, ``x == x``, ``x != x`` are all false. This behavior is compliant with IEEE 754. +* ``None`` and ``NotImplemented`` are singletons. :PEP:`8` advises that + comparisons for singletons should always be done with ``is`` or ``is not``, + never the equality operators. + * Binary sequences (instances of :class:`bytes` or :class:`bytearray`) can be compared within and across their types. They compare lexicographically using the numeric values of their elements. @@ -1436,25 +1446,9 @@ built-in types. :exc:`TypeError`. Sequences compare lexicographically using comparison of corresponding - elements, whereby reflexivity of the elements is enforced. - - In enforcing reflexivity of elements, the comparison of collections assumes - that for a collection element ``x``, ``x == x`` is always true. Based on - that assumption, element identity is compared first, and element comparison - is performed only for distinct elements. This approach yields the same - result as a strict element comparison would, if the compared elements are - reflexive. For non-reflexive elements, the result is different than for - strict element comparison, and may be surprising: The non-reflexive - not-a-number values for example result in the following comparison behavior - when used in a list:: - - >>> nan = float('NaN') - >>> nan is nan - True - >>> nan == nan - False <-- the defined non-reflexive behavior of NaN - >>> [nan] == [nan] - True <-- list enforces reflexivity and tests identity first + elements. The built-in containers typically assume identical objects are + equal to themselves. That lets them bypass equality tests for identical + objects to improve performance and to maintain their internal invariants. Lexicographical comparison between built-in collections works as follows: diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index cc1b2f57a70e3b..7e1e17edb2d8da 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -594,11 +594,9 @@ escape sequences only recognized in string literals fall into the category of unrecognized escapes for bytes literals. .. versionchanged:: 3.6 - Unrecognized escape sequences produce a :exc:`DeprecationWarning`. - - .. versionchanged:: 3.8 - Unrecognized escape sequences produce a :exc:`SyntaxWarning`. In - some future version of Python they will be a :exc:`SyntaxError`. + Unrecognized escape sequences produce a :exc:`DeprecationWarning`. In + a future Python version they will be a :exc:`SyntaxWarning` and + eventually a :exc:`SyntaxError`. Even in a raw literal, quotes can be escaped with a backslash, but the backslash remains in the result; for example, ``r"\""`` is a valid string diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 8bf7eb699c4e72..f41077b0761521 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -22,8 +22,12 @@ from sphinx import addnodes from sphinx.builders import Builder +try: + from sphinx.errors import NoUri +except ImportError: + from sphinx.environment import NoUri from sphinx.locale import translators -from sphinx.util import status_iterator +from sphinx.util import status_iterator, logging from sphinx.util.nodes import split_explicit_title from sphinx.writers.html import HTMLTranslator from sphinx.writers.text import TextWriter, TextTranslator @@ -157,7 +161,7 @@ class AuditEvent(Directive): has_content = True required_arguments = 1 - optional_arguments = 1 + optional_arguments = 2 final_argument_whitespace = True _label = [ @@ -166,21 +170,54 @@ class AuditEvent(Directive): "Raises an :ref:`auditing event ` {name} with arguments {args}.", ] + @property + def logger(self): + cls = type(self) + return logging.getLogger(cls.__module__ + "." + cls.__name__) + def run(self): + name = self.arguments[0] if len(self.arguments) >= 2 and self.arguments[1]: - args = [ - "``{}``".format(a.strip()) - for a in self.arguments[1].strip("'\"").split() - if a.strip() - ] + args = (a.strip() for a in self.arguments[1].strip("'\"").split(",")) + args = [a for a in args if a] else: args = [] label = translators['sphinx'].gettext(self._label[min(2, len(args))]) - text = label.format(name="``{}``".format(self.arguments[0]), - args=", ".join(args)) + text = label.format(name="``{}``".format(name), + args=", ".join("``{}``".format(a) for a in args if a)) - pnode = nodes.paragraph(text, classes=["audit-hook"]) + env = self.state.document.settings.env + if not hasattr(env, 'all_audit_events'): + env.all_audit_events = {} + + new_info = { + 'source': [], + 'args': args + } + info = env.all_audit_events.setdefault(name, new_info) + if info is not new_info: + if not self._do_args_match(info['args'], new_info['args']): + self.logger.warn( + "Mismatched arguments for audit-event {}: {!r} != {!r}" + .format(name, info['args'], new_info['args']) + ) + + ids = [] + try: + target = self.arguments[2].strip("\"'") + except (IndexError, TypeError): + target = None + if not target: + target = "audit_event_{}_{}".format( + re.sub(r'\W', '_', name), + len(info['source']), + ) + ids.append(target) + + info['source'].append((env.docname, target)) + + pnode = nodes.paragraph(text, classes=["audit-hook"], ids=ids) if self.content: self.state.nested_parse(self.content, self.content_offset, pnode) else: @@ -189,6 +226,37 @@ def run(self): return [pnode] + # This list of sets are allowable synonyms for event argument names. + # If two names are in the same set, they are treated as equal for the + # purposes of warning. This won't help if number of arguments is + # different! + _SYNONYMS = [ + {"file", "path", "fd"}, + ] + + def _do_args_match(self, args1, args2): + if args1 == args2: + return True + if len(args1) != len(args2): + return False + for a1, a2 in zip(args1, args2): + if a1 == a2: + continue + if any(a1 in s and a2 in s for s in self._SYNONYMS): + continue + return False + return True + + +class audit_event_list(nodes.General, nodes.Element): + pass + + +class AuditEventListDirective(Directive): + + def run(self): + return [audit_event_list('')] + # Support for documenting decorators @@ -394,7 +462,7 @@ def write(self, *ignored): 'building topics... ', length=len(pydoc_topic_labels)): if label not in self.env.domaindata['std']['labels']: - self.warn('label %r not in documentation' % label) + self.env.logger.warn('label %r not in documentation' % label) continue docname, labelid, sectname = self.env.domaindata['std']['labels'][label] doctree = self.env.get_and_resolve_doctree(docname, self) @@ -458,12 +526,76 @@ def parse_pdb_command(env, sig, signode): return fullname +def process_audit_events(app, doctree, fromdocname): + for node in doctree.traverse(audit_event_list): + break + else: + return + + env = app.builder.env + + table = nodes.table(cols=3) + group = nodes.tgroup( + '', + nodes.colspec(colwidth=30), + nodes.colspec(colwidth=55), + nodes.colspec(colwidth=15), + ) + head = nodes.thead() + body = nodes.tbody() + + table += group + group += head + group += body + + row = nodes.row() + row += nodes.entry('', nodes.paragraph('', nodes.Text('Audit event'))) + row += nodes.entry('', nodes.paragraph('', nodes.Text('Arguments'))) + row += nodes.entry('', nodes.paragraph('', nodes.Text('References'))) + head += row + + for name in sorted(getattr(env, "all_audit_events", ())): + audit_event = env.all_audit_events[name] + + row = nodes.row() + node = nodes.paragraph('', nodes.Text(name)) + row += nodes.entry('', node) + + node = nodes.paragraph() + for i, a in enumerate(audit_event['args']): + if i: + node += nodes.Text(", ") + node += nodes.literal(a, nodes.Text(a)) + row += nodes.entry('', node) + + node = nodes.paragraph() + backlinks = enumerate(sorted(set(audit_event['source'])), start=1) + for i, (doc, label) in backlinks: + if isinstance(label, str): + ref = nodes.reference("", nodes.Text("[{}]".format(i)), internal=True) + try: + ref['refuri'] = "{}#{}".format( + app.builder.get_relative_uri(fromdocname, doc), + label, + ) + except NoUri: + continue + node += ref + row += nodes.entry('', node) + + body += row + + for node in doctree.traverse(audit_event_list): + node.replace_self(table) + + def setup(app): app.add_role('issue', issue_role) app.add_role('source', source_role) app.add_directive('impl-detail', ImplementationDetail) app.add_directive('availability', Availability) app.add_directive('audit-event', AuditEvent) + app.add_directive('audit-event-table', AuditEventListDirective) app.add_directive('deprecated-removed', DeprecatedRemoved) app.add_builder(PydocTopicsBuilder) app.add_builder(suspicious.CheckSuspiciousMarkupBuilder) @@ -478,4 +610,5 @@ def setup(app): app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) app.add_directive('miscnews', MiscNews) + app.connect('doctree-resolved', process_audit_events) return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/suspicious.py b/Doc/tools/extensions/suspicious.py index 494efabc46234e..9e814fb94d2b56 100644 --- a/Doc/tools/extensions/suspicious.py +++ b/Doc/tools/extensions/suspicious.py @@ -115,10 +115,12 @@ def write_doc(self, docname, doctree): def finish(self): unused_rules = [rule for rule in self.rules if not rule.used] if unused_rules: - self.warn('Found %s/%s unused rules:' % - (len(unused_rules), len(self.rules))) - for rule in unused_rules: - self.logger.info(repr(rule)) + self.logger.warning( + 'Found %s/%s unused rules: %s' % ( + len(unused_rules), len(self.rules), + ''.join(repr(rule) for rule in unused_rules), + ) + ) return def check_issue(self, line, lineno, issue): @@ -151,14 +153,15 @@ def report_issue(self, text, lineno, issue): self.any_issue = True self.write_log_entry(lineno, issue, text) if py3: - self.warn('[%s:%d] "%s" found in "%-.120s"' % - (self.docname, lineno, issue, text)) + self.logger.warning('[%s:%d] "%s" found in "%-.120s"' % + (self.docname, lineno, issue, text)) else: - self.warn('[%s:%d] "%s" found in "%-.120s"' % ( - self.docname.encode(sys.getdefaultencoding(),'replace'), - lineno, - issue.encode(sys.getdefaultencoding(),'replace'), - text.strip().encode(sys.getdefaultencoding(),'replace'))) + self.logger.warning( + '[%s:%d] "%s" found in "%-.120s"' % ( + self.docname.encode(sys.getdefaultencoding(),'replace'), + lineno, + issue.encode(sys.getdefaultencoding(),'replace'), + text.strip().encode(sys.getdefaultencoding(),'replace'))) self.app.statuscode = 1 def write_log_entry(self, lineno, issue, text): diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js index 346b31494e60f9..fa298a76b0fe10 100644 --- a/Doc/tools/static/switchers.js +++ b/Doc/tools/static/switchers.js @@ -10,7 +10,8 @@ '(?:release/\\d.\\d[\\x\\d\\.]*)']; var all_versions = { - '3.8': 'dev (3.8)', + '3.9': 'dev (3.9)', + '3.8': 'pre (3.8)', '3.7': '3.7', '3.6': '3.6', '3.5': '3.5', diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index 85263d47c8bba8..d6252ed2ffa24a 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -4,7 +4,7 @@ c-api/sequence,,:i2,del o[i1:i2] c-api/sequence,,:i2,o[i1:i2] c-api/unicode,,:end,str[start:end] c-api/unicode,,:start,unicode[start:start+length] -distutils/examples,274,`,This is the description of the ``foobar`` package. +distutils/examples,267,`,This is the description of the ``foobar`` package. distutils/setupscript,,::, extending/embedding,,:numargs,"if(!PyArg_ParseTuple(args, "":numargs""))" extending/extending,,:myfunction,"PyArg_ParseTuple(args, ""D:myfunction"", &c);" @@ -216,8 +216,6 @@ library/stdtypes,,:len,s[len(s):len(s)] library/stdtypes,,::,>>> y = m[::2] library/stdtypes,,::,>>> z = y[::-2] library/string,,`,"!""#$%&'()*+,-./:;<=>?@[\]^_`{|}~" -library/subprocess,,`,"output=`dmesg | grep hda`" -library/subprocess,,`,"output=`mycmd myarg`" library/tarfile,,:bz2, library/tarfile,,:compression,filemode[:compression] library/tarfile,,:gz, diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html index fca44e9163cac7..21af621efce95b 100644 --- a/Doc/tools/templates/customsourcelink.html +++ b/Doc/tools/templates/customsourcelink.html @@ -4,7 +4,7 @@

{{ _('This Page') }}