config.yml

# Copyright 2019 PrivateStorage.io, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

version: 2.1

orbs:
  # Get easier access to the Windows machine executor.
  win: "circleci/windows@4.1.1"

aliases:
  # Custom checkout step using https URL instead of the ssh one provided by GitHub OAuth
  # This avoids us having to manage keys only to read a public repo - until we have to push?
  # But we have to handle remote branches from external fork (CIRCLE_PR_XXX)
  # and local branches from the project itself (CIRCLE_PROJECT)
  # In addition, the code will differ between bash and powershell
  - &CHECKOUT
    run:
      name: Checkout code
      command: |
        USERNAME=${CIRCLE_PR_USERNAME:-${CIRCLE_PROJECT_USERNAME}}
        REPONAME=${CIRCLE_PR_REPONAME:-${CIRCLE_PROJECT_REPONAME}}
        git clone "https://github.com/${USERNAME}/${REPONAME}.git" .
        git checkout -B "${CIRCLE_BRANCH}" "${CIRCLE_SHA1}"
        TERM=ansi git log -n 1 --pretty=oneline

  - &CHECKOUT_WINDOWS
    run:
      name: Checkout code
      command: |
        $USERNAME = if ($Env:CIRCLE_PR_USERNAME -ne $null) { $Env:CIRCLE_PR_USERNAME } else { $Env:CIRCLE_PROJECT_USERNAME }
        $REPONAME = if ($Env:CIRCLE_PR_REPONAME -ne $null) { $Env:CIRCLE_PR_REPONAME } else { $Env:CIRCLE_PROJECT_REPONAME }
        git clone "https://github.com/$USERNAME/$REPONAME.git" .
        git checkout -B "$Env:CIRCLE_BRANCH" "$Env:CIRCLE_SHA1"
        git log -n 1 --pretty=oneline

  - &PREPARE_VIRTUALENV
    run:
      name: "Prepare virtualenv"
      command: |
        virtualenv venv
        . venv/bin/activate
        pip install --upgrade certifi pip
        pip install ${PIP_REQUIREMENTS}

  - nix_docker: &NIX_DOCKER
      # Run in a highly Nix-capable environment.
      - image: "nixos/nix:2.18.9"

  - nix_environ: &NIX_ENVIRON
      # Let us use features marked "experimental".  For example, most/all of
      # the `nix <subcommand>` forms.
      NIX_CONFIG: "experimental-features = nix-command flakes"

      # Pin a NixOS 21.11 revision.  Most of the software involved in the
      # build process is pinned by nix/sources.json with niv but a few things
      # need to work before we get that far.  This pin is for those things.
      # This pin has no particular bearing on what version of our dependencies
      # we are testing against, what version of Python we support, etc.  It is
      # part of CI infrastructure.
      NIXPKGS: "https://github.com/NixOS/nixpkgs/archive/28abc4e43a24d28729509e2d83f5c4f3b3418189.tar.gz"


jobs:
  documentation:
    docker:
      - image: "circleci/python:3.7"

    environment:
      PIP_REQUIREMENTS: "-r docs/requirements.txt"

    steps:
      - <<: *CHECKOUT

      - <<: *PREPARE_VIRTUALENV

      - run:
          name: "Sphinx Documentation Build"
          command: |
            . venv/bin/activate
            sphinx-build docs/source docs/build

      - store_artifacts:
          path: "docs/build"
          destination: "docs"

  windows-tests:
    parameters:
      py-version:
        type: "string"

    executor:
      # https://circleci.com/developer/orbs/orb/circleci/windows
      name: "win/server-2022"
      # resource class, can be "medium", "large", "xlarge", "2xlarge"
      size: "medium"
      # The default Windows machine image changes from time to time - which
      # often breaks things.  Avoid that.
      version: "2024.04.1"

    steps:
      # Commands are run in a Windows virtual machine environment
      - <<: *CHECKOUT_WINDOWS
      - run:
          name: "Setup Environment"
          command: |
            py -<< parameters.py-version >> -c "# This err-exits if no Python of this version is installed"
            if (-not $?) { choco install python --version=<< parameters.py-version >> --confirm --allow-downgrade }
            py -<< parameters.py-version >> -V
            py -<< parameters.py-version >> -m pip install -v --upgrade pip wheel
            py -<< parameters.py-version >> -m pip install -v . -r requirements/test.in
            py -<< parameters.py-version >> -m pip freeze
      - run:
          name: "Run Tests"
          command: |
            py -<< parameters.py-version >> -m coverage run `
              --debug=config `
              --module twisted.trial `
                --rterrors `
                _zkapauthorizer

      - run:
          name: "Persist Coverage to Workspace"
          command: |
            mkdir -p coverage-workspace
            Copy-Item -Path ".coverage.*" -Destination "coverage-workspace"
            py -<< parameters.py-version >> -m coverage combine
            py -<< parameters.py-version >> -m coverage report

      - persist_to_workspace:
          root: "coverage-workspace"
          paths:
            - "*"

  macos-tests:
    parameters:
      py-version:
        type: "string"
      xcode-version:
        type: "string"

    macos:
      xcode: << parameters.xcode-version >>

    steps:
      - <<: *CHECKOUT

      - run:
          name: "Install Python"
          command: |
            type -p python<< parameters.py-version >> || brew install python@<< parameters.py-version >>
            echo 'export PATH="/usr/local/opt/python@<< parameters.py-version >>/bin:$PATH"' >> "$BASH_ENV"

      - restore_cache:
          keys:
          # when packaging metadata changes, use increasingly general patterns
          # to restore cache
          - pip-packages-v1-{{ .Branch }}-{{ checksum "setup.cfg" }}-{{ checksum "requirements/test.in" }}
          - pip-packages-v1-{{ .Branch }}-{{ checksum "setup.cfg" }}
          - pip-packages-v1-{{ .Branch }}-
          - pip-packages-v1-

      - run:
          name: "Get Pip"
          command: |
            curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
            python<< parameters.py-version >> get-pip.py

      - run:
          name: "Create Virtualenv"
          command: |
            # The CircleCI macOS Python environment has some Python libraries
            # in it which conflict with ZKAPAuthorizer's requirements.  So
            # install virtualenv and use it to create an environment for
            # ZKAPAuthorizer so it can have all its own versions of its
            # dependencies.
            python<< parameters.py-version >> -m venv venv

            # Get the newest pip we can because who knows what came along with
            # that virtualenv.  Also get the wheel package or we won't be able
            # to build any wheels in the next step.
            venv/bin/python -m pip install \
              --find-links file://${PWD}/wheelhouse \
              --upgrade pip wheel

      - run:
          name: "Populate Wheelhouse"
          command: |
            # Get wheels for all the Python packages we depend on - either
            # directly via the ZKAPAuthorizer distutils configuration *or*
            # because some other tool depends on it.  For example, pip has a
            # bunch of dependencies it is nice to have around, and using the
            # wheels depends on the wheel package.
            venv/bin/python -m pip wheel \
              --wheel-dir wheelhouse \
              -r requirements/test.in \
              pip wheel .

      - save_cache:
          paths:
          - "wheelhouse"
          key: pip-packages-v1-{{ .Branch }}-{{ checksum "setup.cfg" }}-{{ checksum "requirements/test.in" }}

      - run:
          name: "Install"
          command: |
            # Now we can install ZKAPAuthorizer and its dependencies and test
            # dependencies into the environment.
            venv/bin/pip install \
              --no-index \
              --find-links file://${PWD}/wheelhouse \
              -r requirements/test.in \
              .

      - run:
          name: "Test"
          command: |
            # The test suite might leak file descriptors.  macOS defaults to a
            # limit of 256.  This should be fixed, but not now ...
            ulimit -Sn 1024

            # Tell coverage in the worker processes where it can find our
            # configuration file.
            export COVERAGE_PROCESS_START=${PWD}/.coveragerc

            # And finally we can run the tests.  We'll run them with 4 jobs
            # because the resource class documented at
            # https://support.circleci.com/hc/en-us/articles/360009144794-macOS-resources
            # says "Medium: 4 vCPUs, 8GB RAM".
            venv/bin/python -m coverage run \
                --debug=config \
                --rcfile=${PWD}/.coveragerc \
                --module twisted.trial \
                  --jobs 4 \
                  --rterrors \
                  _zkapauthorizer

          environment:
            ZKAPAUTHORIZER_HYPOTHESIS_PROFILE: "ci"

      - run:
          name: "Persist Coverage to Workspace"
          command: |
            mkdir -p coverage-workspace
            cp -v .coverage.* coverage-workspace
            venv/bin/python -m coverage combine
            venv/bin/python -m coverage report

      - persist_to_workspace:
          root: "coverage-workspace"
          paths:
            - "*"

  pypi-upload:
    parameters:
      repository:
        type: "string"

    docker:
      - <<: *NIX_DOCKER

    environment:
      <<: *NIX_ENVIRON

      TWINE_REPOSITORY: "<< parameters.repository >>"
      TWINE_USERNAME: "__token__"


    steps:
      # The only reason we need the source for this step is to get
      # nix/twine.nix that defines the shell environment we can use to upload
      # the wheel.
      - <<: *CHECKOUT

      - attach_workspace:
          at: "release-workspace"

      - run:
          name: "Upload Wheel"
          command: |
            set -eux -o pipefail

            if [ "<< parameters.repository >>" = "testpypi" ]; then
              export TWINE_PASSWORD="${TESTPYPI_API_TOKEN}"
            elif [ "<< parameters.repository >>" = "pypi" ]; then
              export TWINE_PASSWORD="${PYPI_API_TOKEN}"
            else
              echo "Unknown repository: << parameters.repository >>"
              exit 1
            fi

            nix run .#twine -- upload --non-interactive release-workspace/*.whl

  build-wheel:
    docker:
      - <<: *NIX_DOCKER

    environment:
      <<: *NIX_ENVIRON

    steps:
      - <<: *CHECKOUT
      - run:
          name: "Build Wheel"
          command: |
            nix build .#wheel
            # Put the wheel in a regular directory so CircleCI doesn't get
            # confused by symlinks when it comes time to upload the wheel as
            # an artifact.
            mkdir wheels
            cp ./result/* wheels/
      - store_artifacts:
          path: "./wheels/"
      - persist_to_workspace:
          root: "./wheels/"
          paths:
            - "*"

  linux-tests: &LINUX_TESTS
    parameters:
      py-version:
        type: "string"
      tahoe-lafs:
        # Identify a Tahoe-LAFS version against which to test.  This must
        # correspond to the `version` property of one of the attrsets defined
        # in `nix/tahoe-version.nix`.
        type: "string"

    docker:
      - <<: *NIX_DOCKER

    # Tahoe-LAFS requires more memory than we get from the default resource
    # class and sometimes we have to build it.
    resource_class: "large"

    environment:
      <<: *NIX_ENVIRON

      # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us
      # to push to CACHIX_NAME.
      CACHIX_NAME: "privatestorage-opensource"

    steps:
      - run: &SETUP_CACHIX
          name: "Set up Cachix"
          command: |
            # Install cachix, the Nix-friendly caching tool.
            nix-env -f $NIXPKGS -iA cachix bash
            # Activate it for "binary substitution".  This sets up
            # configuration that lets Nix download something from the cache
            # instead of building it locally, if possible.
            cachix use "${CACHIX_NAME}"

      - <<: *CHECKOUT

      - run:
          name: "Run Test Suite"
          # If we have to build absolutely everything from scratch or if it
          # takes a while to download pypi-deps-db or some reason then we
          # might be busy for a while.  We use --verbose to try to be a little
          # noisier as we build but who knows what that actually guarantees...
          no_output_timeout: "30m"
          command: |
            # Build one of the test packages defined by the flake.  Choose one
            # that matches our parameters - python version, etc.
            #
            # Also explicitly specify the number of cores to use such that it
            # only slightly exceeds what CircleCI advertises for the resource
            # class (defined above) we're using.  The CircleCI environment
            # looks like it has many more cores than are actually usable by
            # our build.
            source .circleci/lib.sh
            cache_if_able nix build \
              --verbose \
              --cores 5 \
              .#tests-python<< parameters.py-version >>-tahoe_<< parameters.tahoe-lafs >>-ci-cov

      - run:
          name: "Persist Coverage to Workspace"
          command: |
            mkdir -p coverage-workspace
            cp ./result*/.coverage coverage-workspace/.coverage.linux-<< parameters.py-version >>-<< parameters.tahoe-lafs >>

      - persist_to_workspace:
          root: "coverage-workspace"
          paths:
            - "*"

  typecheck:
    <<: *LINUX_TESTS

    steps:
      - run:
          <<: *SETUP_CACHIX
      - <<: *CHECKOUT

      - run:
          name: "Check Black"
          command: |
            source .circleci/lib.sh
            cache_if_able nix run -v .#black -- --check --diff src

      - run:
          name: "Check isort"
          command: |
            source .circleci/lib.sh
            cache_if_able nix run .#isort -- --check --diff src

      - run:
          name: "Check flake8"
          command: |
            source .circleci/lib.sh
            cache_if_able nix run .#flake8 -- src

      - run:
          name: "Run Type Checks"
          command: |
            source .circleci/lib.sh
            cache_if_able nix run .#mypy -- src


  # A job that merges all of the coverage data and generates a report.
  finish-coverage:
    docker:
      - <<: *NIX_DOCKER

    steps:
      - <<: *CHECKOUT
      - attach_workspace:
          at: "coverage-workspace"

      - store_artifacts:
          path: "coverage-workspace"

      - run:
          name: "Finish Coverage Report"
          command: |
            .circleci/finish-coverage-report ./coverage-workspace


workflows:
  version: 2

  # Define a workflow for publishing a release.  Triggering workflows on tags
  # is complex.  For a lot of hints see
  # https://circleci.com/docs/2.0/workflows#executing-workflows-for-a-git-tag
  release:
    jobs:
      - "build-wheel":
          # Documentation states there is a requirement to have a tag filter
          # on a job is required by another job with a tag filter.  Just
          # duplicate the pypi-upload job's tag filter so we don't run if no
          # upload is happening.
          filters:
            tags:
              # Limit the wheel job to running only for release tags -
              # excluding release candidates.  We try to match things like
              # "v1.2.3" and not things like "v1.2.3a5".
              only: "/^v\\d+\\.\\d+\\.\\d+$/"
            branches:
              # And make sure it runs for no other branch pushes.
              ignore: "/.*/"
      - "pypi-upload":
          filters:
            tags:
              # Limit the upload job to running only for release tags.
              only: "/^v\\d+\\.\\d+\\.\\d+$/"
            branches:
              # And make sure it runs for no other branch pushes.
              ignore: "/.*/"
          repository: "pypi"
          requires:
            - "build-wheel"

  # Like the `release` workflow but only run on release candidate tags and
  # upload to Test PyPI.  See `release` workflow for commentary.
  release-candidate:
    jobs:
      - "build-wheel":
          filters:
            tags:
              only: "/^v\\d+\\.\\d+\\.\\d+a\\d+$/"
            branches:
              ignore: "/.*/"
      - "pypi-upload":
          filters:
            tags:
              only: "/^v\\d+\\.\\d+\\.\\d+a\\d+$/"
            branches:
              ignore: "/.*/"
          repository: "testpypi"
          requires:
            - "build-wheel"

  everything:
    jobs:
    - "documentation"
    - "typecheck":
        py-version: "39"
        tahoe-lafs: "1_18_0"
    - "build-wheel"
    - "linux-tests":
        matrix:
          parameters:
            py-version:
              - "39"
              - "310"
              - "311"
              - "312"
            tahoe-lafs:
              - "1_18_0"
              # This is usually not master@HEAD because it is still pinned to
              # a certain revision.  The intent is to update it frequently and
              # discover fixable incompatibilities in small groups and
              # unfixable incompatibilities early enough to prevent them from
              # going into a release.
              - "dev"

    # https://circleci.com/docs/using-macos/
    - "macos-tests":
        matrix:
          parameters:
            py-version:
              - "3.9"
              - "3.10"
            xcode-version:
              - "13.4.1"

    - "windows-tests":
        matrix:
          parameters:
            py-version:
              - "3.9"
              - "3.10"
              - "3.11"
              - "3.12"

    - "finish-coverage":
        # Make sure it depends on all coverage-collecting jobs!
        requires:
          - "linux-tests-39-1_18_0"
          - "linux-tests-39-dev"
          - "linux-tests-310-1_18_0"
          - "linux-tests-310-dev"
          - "macos-tests-3.9-13.4.1"
          - "macos-tests-3.10-13.4.1"
          - "windows-tests-3.9"
          - "windows-tests-3.10"