Merge remote-tracking branch 'upstream/master' into upstream-RossComputerGuy/feat/expose-computefsclosure

Author: roberth

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,6 +15,10 @@ so you understand the process and the expectations.
 - volunteering contributions effectively
 - how to get help and our review process.
 
+PR stuck in review? We have two Nix team meetings per week online that are open for everyone in a jitsi conference:
+
+- https://calendar.google.com/calendar/u/0/[email protected]
+
 -->
 
 ## Motivation

.github/actions/install-nix-action/action.yaml

@@ -4,12 +4,22 @@ inputs:
   dogfood:
     description: "Whether to use Nix installed from the latest artifact from master branch"
     required: true # Be explicit about the fact that we are using unreleased artifacts
+  experimental-installer:
+    description: "Whether to use the experimental installer to install Nix"
+    default: false
+  experimental-installer-version:
+    description: "Version of the experimental installer to use. If `latest`, the newest artifact from the default branch is used."
+    # TODO: This should probably be pinned to a release after https://github.com/NixOS/experimental-nix-installer/pull/49 lands in one
+    default: "latest"
   extra_nix_config:
     description: "Gets appended to `/etc/nix/nix.conf` if passed."
   install_url:
     description: "URL of the Nix installer"
     required: false
     default: "https://releases.nixos.org/nix/nix-2.30.2/install"
+  tarball_url:
+    description: "URL of the Nix tarball to use with the experimental installer"
+    required: false
   github_token:
     description: "Github token"
     required: true
@@ -37,14 +47,74 @@ runs:
 
         gh run download "$RUN_ID" --repo "$DOGFOOD_REPO" -n "$INSTALLER_ARTIFACT" -D "$INSTALLER_DOWNLOAD_DIR"
         echo "installer-path=file://$INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT"
+        TARBALL_PATH="$(find "$INSTALLER_DOWNLOAD_DIR" -name 'nix*.tar.xz' -print | head -n 1)"
+        echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
 
         echo "::notice ::Dogfooding Nix installer from master (https://github.com/$DOGFOOD_REPO/actions/runs/$RUN_ID)"
       env:
         GH_TOKEN: ${{ inputs.github_token }}
         DOGFOOD_REPO: "NixOS/nix"
+    - name: "Gather system info for experimental installer"
+      shell: bash
+      if: ${{ inputs.experimental-installer == 'true' }}
+      run: |
+        echo "::notice Using experimental installer from $EXPERIMENTAL_INSTALLER_REPO (https://github.com/$EXPERIMENTAL_INSTALLER_REPO)"
+
+        if [ "$RUNNER_OS" == "Linux" ]; then
+          EXPERIMENTAL_INSTALLER_SYSTEM="linux"
+          echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV"
+        elif [ "$RUNNER_OS" == "macOS" ]; then
+          EXPERIMENTAL_INSTALLER_SYSTEM="darwin"
+          echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV"
+        else
+          echo "::error ::Unsupported RUNNER_OS: $RUNNER_OS"
+          exit 1
+        fi
+
+        if [ "$RUNNER_ARCH" == "X64" ]; then
+          EXPERIMENTAL_INSTALLER_ARCH=x86_64
+          echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV"
+        elif [ "$RUNNER_ARCH" == "ARM64" ]; then
+          EXPERIMENTAL_INSTALLER_ARCH=aarch64
+          echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV"
+        else
+          echo "::error ::Unsupported RUNNER_ARCH: $RUNNER_ARCH"
+          exit 1
+        fi
+
+        echo "EXPERIMENTAL_INSTALLER_ARTIFACT=nix-installer-$EXPERIMENTAL_INSTALLER_ARCH-$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV"
+      env:
+        EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer"
+    - name: "Download latest experimental installer"
+      shell: bash
+      id: download-latest-experimental-installer
+      if: ${{ inputs.experimental-installer == 'true' && inputs.experimental-installer-version == 'latest' }}
+      run: |
+        RUN_ID=$(gh run list --repo "$EXPERIMENTAL_INSTALLER_REPO" --workflow ci.yml --branch main --status success --json databaseId --jq ".[0].databaseId")
+
+        EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR="$GITHUB_WORKSPACE/$EXPERIMENTAL_INSTALLER_ARTIFACT"
+        mkdir -p "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR"
+
+        gh run download "$RUN_ID" --repo "$EXPERIMENTAL_INSTALLER_REPO" -n "$EXPERIMENTAL_INSTALLER_ARTIFACT" -D "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR"
+        # Executable permissions are lost in artifacts
+        find $EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR -type f -exec chmod +x {} +
+        echo "installer-path=$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT"
+      env:
+        GH_TOKEN: ${{ inputs.github_token }}
+        EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer"
     - uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1
+      if: ${{ inputs.experimental-installer != 'true' }}
       with:
         # Ternary operator in GHA: https://www.github.com/actions/runner/issues/409#issuecomment-752775072
         install_url: ${{ inputs.dogfood == 'true' && format('{0}/install', steps.download-nix-installer.outputs.installer-path) || inputs.install_url }}
         install_options: ${{ inputs.dogfood == 'true' && format('--tarball-url-prefix {0}', steps.download-nix-installer.outputs.installer-path) || '' }}
         extra_nix_config: ${{ inputs.extra_nix_config }}
+    - uses: DeterminateSystems/nix-installer-action@786fff0690178f1234e4e1fe9b536e94f5433196 # v20
+      if: ${{ inputs.experimental-installer == 'true' }}
+      with:
+        diagnostic-endpoint: ""
+        # TODO: It'd be nice to use `artifacts.nixos.org` for both of these, maybe through an `/experimental-installer/latest` endpoint? or `/commit/<hash>`?
+        local-root: ${{ inputs.experimental-installer-version == 'latest' && steps.download-latest-experimental-installer.outputs.installer-path || '' }}
+        source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }}
+        nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }}
+        extra-conf: ${{ inputs.extra_nix_config }}

.github/workflows/backport.yml

@@ -0,0 +1,37 @@
+name: Backport
+on:
+  pull_request_target:
+    types: [closed, labeled]
+permissions:
+  contents: read
+jobs:
+  backport:
+    name: Backport Pull Request
+    permissions:
+      # for korthout/backport-action
+      contents: write
+      pull-requests: write
+    if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name))
+    runs-on: ubuntu-24.04-arm
+    steps:
+      - name: Generate GitHub App token
+        id: generate-token
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ vars.CI_APP_ID }}
+          private-key: ${{ secrets.CI_APP_PRIVATE_KEY }}
+      - uses: actions/checkout@v5
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          # required to find all branches
+          fetch-depth: 0
+      - name: Create backport PRs
+        uses: korthout/backport-action@d07416681cab29bf2661702f925f020aaa962997 # v3.4.1
+        id: backport
+        with:
+          # Config README: https://github.com/korthout/backport-action#backport-action
+          github_token: ${{ steps.generate-token.outputs.token }}
+          github_workspace: ${{ github.workspace }}
+          auto_merge_enabled: true
+          pull_description: |-
+            Automatic backport to `${target_branch}`, triggered by a label in #${pull_number}.

.github/workflows/ci.yml

@@ -4,6 +4,8 @@ on:
   pull_request:
   merge_group:
   push:
+    branches:
+      - master
   workflow_dispatch:
     inputs:
       dogfood:
@@ -29,7 +31,32 @@ jobs:
         github_token: ${{ secrets.GITHUB_TOKEN }}
     - run: nix flake show --all-systems --json
 
+  pre-commit-checks:
+    name: pre-commit checks
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v5
+      - uses: ./.github/actions/install-nix-action
+        with:
+          dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }}
+          extra_nix_config: experimental-features = nix-command flakes
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: ./ci/gha/tests/pre-commit-checks
+
+  basic-checks:
+    name: aggregate basic checks
+    if: ${{ always() }}
+    runs-on: ubuntu-24.04
+    needs: [pre-commit-checks, eval]
+    steps:
+      - name: Exit with any errors
+        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
+        run: |
+          exit 1
+
   tests:
+    needs: basic-checks
     strategy:
       fail-fast: false
       matrix:
@@ -40,18 +67,42 @@ jobs:
             instrumented: false
             primary: true
             stdenv: stdenv
+            withAWS: true
+            withCurlS3: false
+          # TODO: remove once curl-based-s3 fully lands
+          - scenario: on ubuntu (no s3)
+            runs-on: ubuntu-24.04
+            os: linux
+            instrumented: false
+            primary: false
+            stdenv: stdenv
+            withAWS: false
+            withCurlS3: false
+          # TODO: remove once curl-based-s3 fully lands
+          - scenario: on ubuntu (curl s3)
+            runs-on: ubuntu-24.04
+            os: linux
+            instrumented: false
+            primary: false
+            stdenv: stdenv
+            withAWS: false
+            withCurlS3: true
           - scenario: on macos
             runs-on: macos-14
             os: darwin
             instrumented: false
             primary: true
             stdenv: stdenv
+            withAWS: true
+            withCurlS3: false
           - scenario: on ubuntu (with sanitizers / coverage)
             runs-on: ubuntu-24.04
             os: linux
             instrumented: true
             primary: false
             stdenv: clangStdenv
+            withAWS: true
+            withCurlS3: false
     name: tests ${{ matrix.scenario }}
     runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 60
@@ -74,7 +125,17 @@ jobs:
       run: |
         nix build --file ci/gha/tests/wrapper.nix componentTests -L \
           --arg withInstrumentation ${{ matrix.instrumented }} \
-          --argstr stdenv "${{ matrix.stdenv }}"
+          --argstr stdenv "${{ matrix.stdenv }}" \
+          ${{ format('--arg withAWS {0}', matrix.withAWS) }} \
+          ${{ format('--arg withCurlS3 {0}', matrix.withCurlS3) }}
+    - name: Run VM tests
+      run: |
+        nix build --file ci/gha/tests/wrapper.nix vmTests -L \
+          --arg withInstrumentation ${{ matrix.instrumented }} \
+          --argstr stdenv "${{ matrix.stdenv }}" \
+          ${{ format('--arg withAWS {0}', matrix.withAWS) }} \
+          ${{ format('--arg withCurlS3 {0}', matrix.withCurlS3) }}
+      if: ${{ matrix.os == 'linux' }}
     - name: Run flake checks and prepare the installer tarball
       run: |
         ci/gha/tests/build-checks
@@ -85,6 +146,8 @@ jobs:
         nix build --file ci/gha/tests/wrapper.nix codeCoverage.coverageReports -L \
           --arg withInstrumentation ${{ matrix.instrumented }} \
           --argstr stdenv "${{ matrix.stdenv }}" \
+          ${{ format('--arg withAWS {0}', matrix.withAWS) }} \
+          ${{ format('--arg withCurlS3 {0}', matrix.withCurlS3) }} \
           --out-link coverage-reports
         cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY
       if: ${{ matrix.instrumented }}
@@ -110,9 +173,19 @@ jobs:
           - scenario: on ubuntu
             runs-on: ubuntu-24.04
             os: linux
+            experimental-installer: false
           - scenario: on macos
             runs-on: macos-14
             os: darwin
+            experimental-installer: false
+          - scenario: on ubuntu (experimental)
+            runs-on: ubuntu-24.04
+            os: linux
+            experimental-installer: true
+          - scenario: on macos (experimental)
+            runs-on: macos-14
+            os: darwin
+            experimental-installer: true
     name: installer test ${{ matrix.scenario }}
     runs-on: ${{ matrix.runs-on }}
     steps:
@@ -124,11 +197,22 @@ jobs:
         path: out
     - name: Looking up the installer tarball URL
       id: installer-tarball-url
-      run: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
+      run: |
+        echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
+        TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
+        echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
     - uses: cachix/install-nix-action@v31
+      if: ${{ !matrix.experimental-installer }}
       with:
         install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
         install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }}
+    - uses: ./.github/actions/install-nix-action
+      if: ${{ matrix.experimental-installer }}
+      with:
+        dogfood: false
+        experimental-installer: true
+        tarball_url: ${{ steps.installer-tarball-url.outputs.tarball-path }}
+        github_token: ${{ secrets.GITHUB_TOKEN }}
     - run: sudo apt install fish zsh
       if: matrix.os == 'linux'
     - run: brew install fish
@@ -160,7 +244,7 @@ jobs:
           echo "docker=${{ env._DOCKER_SECRETS != '' }}" >> $GITHUB_OUTPUT
 
   docker_push_image:
-    needs: [tests, vm_tests, check_secrets]
+    needs: [tests, check_secrets]
     permissions:
       contents: read
       packages: write
@@ -213,27 +297,8 @@ jobs:
         docker tag nix:$NIX_VERSION $IMAGE_ID:master
         docker push $IMAGE_ID:master
 
-  vm_tests:
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v5
-      - uses: ./.github/actions/install-nix-action
-        with:
-          dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }}
-          extra_nix_config:
-            experimental-features = nix-command flakes
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: DeterminateSystems/magic-nix-cache-action@main
-      - run: |
-          nix build -L \
-            .#hydraJobs.tests.functional_user \
-            .#hydraJobs.tests.githubFlakes \
-            .#hydraJobs.tests.nix-docker \
-            .#hydraJobs.tests.tarballFlakes \
-            ;
-
   flake_regressions:
-    needs: vm_tests
+    needs: tests
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout nix