diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 68eef0e5..d86dcf2b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -56,7 +56,7 @@ jobs: path: target/release/pythonbuild image: - if: ${{ needs.generate-matrix.outputs.any_builds == 'true' }} + if: ${{ needs.generate-matrix.outputs.pythonbuild_changed == 'true' || needs.generate-matrix.outputs.any_builds == 'true' || github.ref == 'refs/heads/main' }} needs: - generate-matrix strategy: diff --git a/.github/workflows/security-3rd-party-pr-checks.yml b/.github/workflows/security-3rd-party-pr-checks.yml new file mode 100644 index 00000000..5337dabb --- /dev/null +++ b/.github/workflows/security-3rd-party-pr-checks.yml @@ -0,0 +1,10 @@ +name: security-3rd-party-pr-checks +on: + # Allow for manual run of security workflows + workflow_dispatch: + # Scan changed files in PRs (diff-aware scanning): + pull_request: {} +jobs: + running-3rd-party-pr-security-checks: + uses: verkada/securitybots/.github/workflows/3rd-party-pr-checks.yml@main + secrets: inherit diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 94bdc048..30f58786 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -1,9 +1,7 @@ name: windows on: - push: - branches: [main] - pull_request: + workflow_dispatch: # Only run manually concurrency: group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} diff --git a/Justfile b/Justfile index c9b8167f..11180753 100644 --- a/Justfile +++ b/Justfile @@ -28,15 +28,15 @@ cat-python-json archive: # Download release artifacts from GitHub Actions release-download-distributions token commit: mkdir -p dist - cargo run --release -- fetch-release-distributions --token {{token}} --commit {{commit}} --dest dist + cargo run --release -- fetch-release-distributions --org verkada --token {{token}} --commit {{commit}} --dest dist # Upload release artifacts to a GitHub release. release-upload-distributions token datetime tag: - cargo run --release -- upload-release-distributions --token {{token}} --datetime {{datetime}} --tag {{tag}} --dist dist + cargo run --release -- upload-release-distributions --org verkada --token {{token}} --datetime {{datetime}} --tag {{tag}} --dist dist # "Upload" release artifacts to a GitHub release in dry-run mode (skip upload). release-upload-distributions-dry-run token datetime tag: - cargo run --release -- upload-release-distributions --token {{token}} --datetime {{datetime}} --tag {{tag}} --dist dist -n + cargo run --release -- upload-release-distributions --org verkada --token {{token}} --datetime {{datetime}} --tag {{tag}} --dist dist -n # Promote a tag to "latest" by pushing to the `latest-release` branch. release-set-latest-release tag: diff --git a/ci-targets.yaml b/ci-targets.yaml index 0662bbee..417333ea 100644 --- a/ci-targets.yaml +++ b/ci-targets.yaml @@ -4,7 +4,6 @@ darwin: aarch64-apple-darwin: arch: aarch64 python_versions: - - "3.9" - "3.10" - "3.11" - "3.12" @@ -22,7 +21,6 @@ darwin: x86_64-apple-darwin: arch: x86_64 python_versions: - - "3.9" - "3.10" - "3.11" - "3.12" @@ -42,110 +40,6 @@ linux: arch: aarch64 libc: gnu python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - pgo+lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+pgo+lto - minimum-python-version: "3.13" - - armv7-unknown-linux-gnueabi: - arch: armv7 - libc: gnu - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - - armv7-unknown-linux-gnueabihf: - arch: armv7 - libc: gnu - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - - s390x-unknown-linux-gnu: - arch: s390x - libc: gnu - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - - ppc64le-unknown-linux-gnu: - arch: ppc64le - libc: gnu - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - - riscv64-unknown-linux-gnu: - arch: riscv64 - libc: gnu - python_versions: - - "3.9" - "3.10" - "3.11" - "3.12" @@ -166,49 +60,6 @@ linux: arch: x86_64 libc: gnu python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - pgo+lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+pgo+lto - minimum-python-version: "3.13" - run: true - - x86_64_v2-unknown-linux-gnu: - arch: x86_64 - arch_variant: v2 - libc: gnu - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - pgo+lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+pgo+lto - minimum-python-version: "3.13" - run: true - - x86_64_v3-unknown-linux-gnu: - arch: x86_64 - arch_variant: v3 - libc: gnu - python_versions: - - "3.9" - "3.10" - "3.11" - "3.12" @@ -223,206 +74,3 @@ linux: - freethreaded+pgo+lto minimum-python-version: "3.13" run: true - - x86_64_v4-unknown-linux-gnu: - arch: x86_64 - arch_variant: v4 - libc: gnu - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - pgo+lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+pgo+lto - minimum-python-version: "3.13" - run: true - - x86_64-unknown-linux-musl: - arch: x86_64 - libc: musl - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug+static - - noopt+static - - lto+static - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - run: true - - x86_64_v2-unknown-linux-musl: - arch: x86_64 - arch_variant: v2 - libc: musl - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug+static - - noopt+static - - lto+static - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - run: true - - x86_64_v3-unknown-linux-musl: - arch: x86_64 - arch_variant: v3 - libc: musl - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug+static - - noopt+static - - lto+static - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - run: true - - x86_64_v4-unknown-linux-musl: - arch: x86_64 - arch_variant: v4 - libc: musl - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug+static - - noopt+static - - lto+static - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - run: true - - aarch64-unknown-linux-musl: - arch: aarch64 - libc: musl - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - # TODO: Static support is current blocked by some compiler-rt linking issues - # - debug+static - # - noopt+static - # - lto+static - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - run: true - -windows: - i686-pc-windows-msvc: - arch: x86 - vcvars: vcvars32.bat - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - pgo - build_options_conditional: - - options: - - freethreaded+pgo - minimum-python-version: "3.13" - - x86_64-pc-windows-msvc: - arch: x86_64 - vcvars: vcvars64.bat - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - pgo - build_options_conditional: - - options: - - freethreaded+pgo - minimum-python-version: "3.13" - - aarch64-pc-windows-msvc: - arch: aarch64 - vcvars: vcvarsamd64_arm64.bat - python_versions: - # On 3.9 / 3.10, `_tkinter` is failing to be included in the build - # - "3.9" - # - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - pgo - build_options_conditional: - - options: - - freethreaded+pgo - minimum-python-version: "3.13" diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index d21533a8..cb0cbce3 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -44,7 +44,7 @@ sed "${sed_args[@]}" "s|/tools/host|${TOOLS_PATH}/host|g" ${TOOLS_PATH}/host/sha # We force linking of external static libraries by removing the shared # libraries. This is hacky. But we're building in a temporary container # and it gets the job done. -find ${TOOLS_PATH}/deps -name '*.so*' -a \! \( -name 'libtcl*.so*' -or -name 'libtk*.so*' \) -exec rm {} \; +find ${TOOLS_PATH}/deps -name '*.so*' ! -name 'libtcl*.so*' ! -name 'libtk*.so*' ! -name 'fips.so' -exec rm {} \; tar -xf Python-${PYTHON_VERSION}.tar.xz @@ -1278,6 +1278,13 @@ if [ -d "${TOOLS_PATH}/deps/usr/share/terminfo" ]; then cp -av ${TOOLS_PATH}/deps/usr/share/terminfo ${ROOT}/out/python/install/share/ fi +# Copy files required to enable FIPS if enabled. +if [ -f ${TOOLS_PATH}/deps/fipsmodule.cnf ]; then + mkdir -p ${ROOT}/out/python/install/share/ssl + cp -rv ${TOOLS_PATH}/deps/lib/ossl-modules ${ROOT}/out/python/install/share/ssl + cp -av ${TOOLS_PATH}/deps/fipsmodule.cnf ${ROOT}/out/python/install/share/ssl +fi + # config.c defines _PyImport_Inittab and extern references to modules, which # downstream consumers may want to strip. We bundle config.c and config.c.in so # a custom one can be produced downstream. diff --git a/cpython-unix/build-openssl-3.0.sh b/cpython-unix/build-openssl-3.0.sh index 1d1f9134..74526def 100755 --- a/cpython-unix/build-openssl-3.0.sh +++ b/cpython-unix/build-openssl-3.0.sh @@ -16,18 +16,23 @@ pushd openssl-${OPENSSL_3_0_VERSION} # Otherwise it gets set to /tools/deps/ssl by default. case "${TARGET_TRIPLE}" in *apple*) - EXTRA_FLAGS="--openssldir=/private/etc/ssl" + OPENSSL_DIR=/private/etc/ssl ;; *) - EXTRA_FLAGS="--openssldir=/etc/ssl" + OPENSSL_DIR=/etc/ssl ;; esac +EXTRA_FLAGS="--openssldir=${OPENSSL_DIR}" +EXTRA_INSTALL_FLAGS="" # musl is missing support for various primitives. # TODO disable secure memory is a bit scary. We should look into a proper # workaround. if [ "${CC}" = "musl-clang" ]; then EXTRA_FLAGS="${EXTRA_FLAGS} no-async -DOPENSSL_NO_ASYNC -D__STDC_NO_ATOMICS__=1 no-engine -DOPENSSL_NO_SECURE_MEMORY" +else + EXTRA_INSTALL_FLAGS="install_fips" + EXTRA_FLAGS="${EXTRA_FLAGS} enable-fips" fi # The -arch cflags confuse Configure. And OpenSSL adds them anyway. @@ -35,6 +40,10 @@ fi EXTRA_TARGET_CFLAGS=${EXTRA_TARGET_CFLAGS/\-arch arm64/} EXTRA_TARGET_CFLAGS=${EXTRA_TARGET_CFLAGS/\-arch x86_64/} +# With -fvisibility=hidden, OSSL_provider_init symbol is not exported in fips module preventing it from loaded +# OSSL_provider_init is supposed to be `extern` so it should not happen but I can't find a more targeted solution +# at the moment. +EXTRA_TARGET_CFLAGS=${EXTRA_TARGET_CFLAGS//-fvisibility=hidden/} EXTRA_FLAGS="${EXTRA_FLAGS} ${EXTRA_TARGET_CFLAGS}" /usr/bin/perl ./Configure \ @@ -47,4 +56,9 @@ EXTRA_FLAGS="${EXTRA_FLAGS} ${EXTRA_TARGET_CFLAGS}" ${EXTRA_FLAGS} make -j ${NUM_CPUS} -make -j ${NUM_CPUS} install_sw install_ssldirs DESTDIR=${ROOT}/out +make -j ${NUM_CPUS} install_sw install_ssldirs ${EXTRA_INSTALL_FLAGS} DESTDIR=${ROOT}/out + +if [ -f ${ROOT}/out${OPENSSL_DIR}/fipsmodule.cnf ]; then + # install_fips does not use DESTDIR. we need to copy it so it gets added to the archive. + cp ${ROOT}/out${OPENSSL_DIR}/fipsmodule.cnf ${ROOT}/out/tools/deps/fipsmodule.cnf +fi diff --git a/src/github.rs b/src/github.rs index 5db22472..a90b9612 100644 --- a/src/github.rs +++ b/src/github.rs @@ -195,7 +195,6 @@ pub async fn command_fetch_release_distributions(args: &ArgMatches) -> Result<() wf.path.as_str(), ".github/workflows/macos.yml" | ".github/workflows/linux.yml" - | ".github/workflows/windows.yml" ) { workflow_names.insert(wf.id, wf.name); @@ -226,6 +225,7 @@ pub async fn command_fetch_release_distributions(args: &ArgMatches) -> Result<() workflows .list_runs(format!("{workflow_id}")) .event("push") + .event("pull_request") .status("success") .send() .await? diff --git a/src/release.rs b/src/release.rs index 4c123f61..564d544e 100644 --- a/src/release.rs +++ b/src/release.rs @@ -145,19 +145,6 @@ pub static RELEASE_TRIPLES: Lazy> = Lazy:: // Linux. let linux_suffixes_pgo = vec!["debug", "pgo+lto"]; let linux_suffixes_nopgo = vec!["debug", "lto", "noopt"]; - let linux_suffixes_musl = vec![ - "debug", - "lto", - "noopt", - "debug+static", - "lto+static", - "noopt+static", - ]; - let linux_suffixes_musl_freethreaded = vec![ - "freethreaded+debug", - "freethreaded+lto", - "freethreaded+noopt", - ]; let linux_suffixes_pgo_freethreaded = vec!["freethreaded+debug", "freethreaded+pgo+lto"]; let linux_suffixes_nopgo_freethreaded = vec![ "freethreaded+debug", @@ -178,70 +165,6 @@ pub static RELEASE_TRIPLES: Lazy> = Lazy:: }, ); - h.insert( - "ppc64le-unknown-linux-gnu", - TripleRelease { - suffixes: linux_suffixes_nopgo.clone(), - install_only_suffix: "lto", - python_version_requirement: Some(VersionSpecifier::from_str(">=3.9").unwrap()), - conditional_suffixes: vec![ConditionalSuffixes { - python_version_requirement: VersionSpecifier::from_str(">=3.13").unwrap(), - suffixes: linux_suffixes_nopgo_freethreaded.clone(), - }], - }, - ); - - h.insert( - "riscv64-unknown-linux-gnu", - TripleRelease { - suffixes: linux_suffixes_nopgo.clone(), - install_only_suffix: "lto", - python_version_requirement: Some(VersionSpecifier::from_str(">=3.9").unwrap()), - conditional_suffixes: vec![ConditionalSuffixes { - python_version_requirement: VersionSpecifier::from_str(">=3.13").unwrap(), - suffixes: linux_suffixes_nopgo_freethreaded.clone(), - }], - }, - ); - - h.insert( - "s390x-unknown-linux-gnu", - TripleRelease { - suffixes: linux_suffixes_nopgo.clone(), - install_only_suffix: "lto", - python_version_requirement: Some(VersionSpecifier::from_str(">=3.9").unwrap()), - conditional_suffixes: vec![ConditionalSuffixes { - python_version_requirement: VersionSpecifier::from_str(">=3.13").unwrap(), - suffixes: linux_suffixes_nopgo_freethreaded.clone(), - }], - }, - ); - - h.insert( - "armv7-unknown-linux-gnueabi", - TripleRelease { - suffixes: linux_suffixes_nopgo.clone(), - install_only_suffix: "lto", - python_version_requirement: Some(VersionSpecifier::from_str(">=3.9").unwrap()), - conditional_suffixes: vec![ConditionalSuffixes { - python_version_requirement: VersionSpecifier::from_str(">=3.13").unwrap(), - suffixes: linux_suffixes_nopgo_freethreaded.clone(), - }], - }, - ); - - h.insert( - "armv7-unknown-linux-gnueabihf", - TripleRelease { - suffixes: linux_suffixes_nopgo.clone(), - install_only_suffix: "lto", - python_version_requirement: Some(VersionSpecifier::from_str(">=3.9").unwrap()), - conditional_suffixes: vec![ConditionalSuffixes { - python_version_requirement: VersionSpecifier::from_str(">=3.13").unwrap(), - suffixes: linux_suffixes_nopgo_freethreaded.clone(), - }], - }, - ); h.insert( "x86_64-unknown-linux-gnu", diff --git a/src/validation.rs b/src/validation.rs index 33e61e2e..8a06bf88 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -1899,6 +1899,12 @@ fn validate_distribution( let mut entry = entry.map_err(|e| anyhow!("failed to iterate over archive: {}", e))?; let path = entry.path()?.to_path_buf(); + if let Some(file_name) = path.file_name() { + if file_name == "fips.dylib" || file_name == "fips.so" { + continue + } + } + seen_paths.insert(path.clone()); if let Some(link_name) = entry.link_name()? {