Add Poetry Support

.github/workflows/run_checks.yaml

@@ -21,7 +21,7 @@ jobs:
   integration_tests:
     if: github.event.pull_request.head.repo.owner.login == 'apify'
     name: Run integration tests
-    needs: [lint_and_type_checks, unit_tests]
+    needs: [lint_and_type_checks, unit_tests, unit_tests_poetry]
     uses: ./.github/workflows/integration_tests.yaml
     secrets: inherit
 

.github/workflows/unit_tests.yaml

@@ -8,8 +8,8 @@ jobs:
     name: Run unit tests
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest]
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -25,8 +25,19 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install dependencies
-        run: make install-dev
-
-      - name: Run unit tests
-        run: make unit-tests
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v2
+      - name: Setup a local virtual environment (if no poetry.toml file)
+        run: |
+          poetry config virtualenvs.create true --local
+          poetry config virtualenvs.in-project true --local
+      - uses: actions/cache@v3
+        name: Define a cache for the virtual environment based on the dependencies lock file
+        with:
+          path: ./.venv
+          key: venv-${{ hashFiles('poetry.lock') }}
+      - name: Install the project dependencies
+        run: poetry install
+      - name: Run the automated tests (for example)
+        run: poetry run pytest tests/unit
+    continue-on-error: true

CONTRIBUTING.md

@@ -8,45 +8,64 @@ It is recommended to set up a virtual environment while developing this package
 however, due to the many varied ways Python can be installed and virtual environments can be set up,
 this is left up to the developers to do themselves.
 
-One recommended way is with the built-in `venv` module:
+## Dependencies
+
+To install this package and its development dependencies, run:
 
 ```bash
-python3 -m venv .venv
-source .venv/bin/activate
+make install-dev
 ```
 
-To improve on the experience, you can use [pyenv](https://github.com/pyenv/pyenv) to have an environment with a pinned Python version,
-and [direnv](https://github.com/direnv/direnv) to automatically activate/deactivate the environment when you enter/exit the project folder.
-
-## Dependencies
-
-To install this package and its development dependencies, run `make install-dev`.
-
 ## Code checking
 
-To run all our code checking tools together, just run `make check-code`.
+To run all our code checking tools together, just run:
+
+```bash
+make check-code
+```
 
 ### Linting
 
 We use [ruff](https://docs.astral.sh/ruff/) for linting to to analyze the code for potential issues and enforce
 uniformed code style. See the `pyproject.toml` for its configuration. To run the linting, just run `make lint`.
 
+```bash
+make lint
+```
+
 ### Formatting
 
 We use [ruff](https://docs.astral.sh/ruff/) for automated code formatting. It formats the code to follow uniformed
 code style and addresses auto-fixable linting issues. See the `pyproject.toml` for its configuration. To run
-the formatting, just run `make format`.
+the formatting, just run:
+
+```bash
+make format
+```
 
 ### Type checking
 
 We use [mypy](https://mypy.readthedocs.io/en/stable/) for type checking. See the `mypy.ini` for its configuration.
-To run the type checking, just run `make type-check`.
+To run the type checking, just run:
+
+```bash
+make type-check
+```
 
 ### Unit tests
 
 We use [pytest](https://docs.pytest.org/) as a testing framework with many plugins. See the `pyproject.toml` for
-both its configuration and the list of installed plugins. To run unit tests execute `make unit-tests`. To run unit
-tests with HTML coverage report execute `make unit-tests-cov`.
+both its configuration and the list of installed plugins. To run unit tests execute:
+
+```bash
+make unit-tests
+```
+
+with coverage report:
+
+```bash
+make unit-tests-cov
+```
 
 ## Integration tests
 
@@ -59,11 +78,14 @@ the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL to t
 
 ## Documentation
 
-We use the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting the code.
+We use the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for
+documenting the code.
 We document every user-facing class or method, and enforce that using the flake8-docstrings library.
 
-The documentation is then rendered from the docstrings in the code, using `pydoc-markdown` and some heavy post-processing,
-and from Markdown documents in the `docs` folder in the `docs` branch, and then rendered using Docusaurus and published to GitHub pages.
+The documentation is then rendered from the docstrings in the code, using `pydoc-markdown` and some heavy
+post-processing,
+and from Markdown documents in the `docs` folder in the `docs` branch, and then rendered using Docusaurus and published
+to GitHub pages.
 
 ## Release process
 
@@ -72,31 +94,38 @@ Publishing new versions to [PyPI](https://pypi.org/project/apify) happens automa
 On each commit to the `master` branch, a new beta release is published, taking the version number from `pyproject.toml`
 and automatically incrementing the beta version suffix by 1 from the last beta release published to PyPI.
 
-A stable version is published when a new release is created using GitHub Releases, again taking the version number from `pyproject.toml`.
+A stable version is published when a new release is created using GitHub Releases, again taking the version number
+from `pyproject.toml`.
 The built package assets are automatically uploaded to the GitHub release.
 
-If there is already a stable version with the same version number as in `pyproject.toml` published to PyPI, the publish process fails,
+If there is already a stable version with the same version number as in `pyproject.toml` published to PyPI, the publish
+process fails,
 so don't forget to update the version number before releasing a new version.
 The release process also fails when the released version is not described in `CHANGELOG.md`,
 so don't forget to describe the changes in the new version there.
 
 ### Beta release checklist
 
-Beta release happens automatically after you merge a pull request or add a direct commit to the master branch. Before you do that check the following:
+Beta release happens automatically after you merge a pull request or add a direct commit to the master branch. Before
+you do that check the following:
 
-- Make sure that in the [pyproject.toml](https://github.com/apify/apify-sdk-python/blob/master/pyproject.toml) a project version is set to the latest non-published version.
-- Describe your changes to the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md) in the section with the latest non-published version.
+- Make sure that in the [pyproject.toml](https://github.com/apify/apify-sdk-python/blob/master/pyproject.toml) a project
+  version is set to the latest non-published version.
+- Describe your changes to the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md) in the
+  section with the latest non-published version.
 
 ### Production release checklist
 
 Production release happens after the GitHub release is created. Before you do that check the following:
 
 - Make sure that the beta release with the latest commit is successfully deployed.
-- Make sure that all the changes that happened from the last production release are described in the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md).
+- Make sure that all the changes that happened from the last production release are described in
+  the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md).
 - When drafting a new GitHub release:
     - Create a new tag in the format of `v1.2.3` targeting the master branch.
     - Fill in the release title in the format of `1.2.3`.
-    - Copy the changes from the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md) and paste them into the release description.
+    - Copy the changes from the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md) and
+      paste them into the release description.
     - Check the "Set as the latest release" option.
 
 ## Maintanance
@@ -108,29 +137,29 @@ Production release happens after the GitHub release is created. Before you do th
 ### Adding support for a new Python version
 
 1) Firstly, ensure that the package (
-    [apify-sdk-python](https://github.com/apify/apify-sdk-python),
-    [apify-client-python](https://github.com/apify/apify-client-python),
-    [apify-shared-python](https://github.com/apify/apify-shared-python)
-) is compatible with the new Python version. Both in our code base and
-the dependencies we use. Then, release a new version of the package.
+   [apify-sdk-python](https://github.com/apify/apify-sdk-python),
+   [apify-client-python](https://github.com/apify/apify-client-python),
+   [apify-shared-python](https://github.com/apify/apify-shared-python)
+   ) is compatible with the new Python version. Both in our code base and
+   the dependencies we use. Then, release a new version of the package.
     - For inspiration, see the PR
-    [apify/apify-sdk-python#121](https://github.com/apify/apify-sdk-python/pull/121),
-    where support for Python 3.12 was added to the Apify Python SDK.
+      [apify/apify-sdk-python#121](https://github.com/apify/apify-sdk-python/pull/121),
+      where support for Python 3.12 was added to the Apify Python SDK.
 
 2) Next, build and publish the new versions of Python base Docker images.
     - For inspiration, see the PR
-    [apify/apify-actor-docker#112](https://github.com/apify/apify-actor-docker/pull/112),
-    where support for Python 3.12 was added.
+      [apify/apify-actor-docker#112](https://github.com/apify/apify-actor-docker/pull/112),
+      where support for Python 3.12 was added.
     - Apify base Docker images are built using GitHub Actions, accessible at
-    [apify/apify-actor-docker/actions](https://github.com/apify/apify-actor-docker/actions).
+      [apify/apify-actor-docker/actions](https://github.com/apify/apify-actor-docker/actions).
 
 3) Integrate the new Python version into the CI/CD workflows
-of existing Python projects (
-    [apify-sdk-python](https://github.com/apify/apify-sdk-python),
-    [apify-client-python](https://github.com/apify/apify-client-python),
-    [apify-shared-python](https://github.com/apify/apify-shared-python),
-    [actor-templates](https://github.com/apify/actor-templates)
-).
+   of existing Python projects (
+   [apify-sdk-python](https://github.com/apify/apify-sdk-python),
+   [apify-client-python](https://github.com/apify/apify-client-python),
+   [apify-shared-python](https://github.com/apify/apify-shared-python),
+   [actor-templates](https://github.com/apify/actor-templates)
+   ).
     - For inspiration, see the PR
-    [apify/apify-sdk-python#124](https://github.com/apify/apify-sdk-python/pull/124),
-    where support for Python 3.12 was added to the CI/CD of the Apify Python SDK.
+      [apify/apify-sdk-python#124](https://github.com/apify/apify-sdk-python/pull/124),
+      where support for Python 3.12 was added to the CI/CD of the Apify Python SDK.

Makefile

@@ -9,39 +9,41 @@ clean:
 	rm -rf build dist .mypy_cache .pytest_cache src/*.egg-info __pycache__
 
 install-dev:
-	python3 -m pip install --upgrade pip
-	pip install --no-cache-dir -e ".[dev,scrapy]"
-	pre-commit install
+	pip install poetry
+	poetry config virtualenvs.create true --local
+	poetry config virtualenvs.in-project true --local
+	poetry install --all-extras
+	poetry run pre-commit install
 
 build:
-	python3 -m build
+	poetry run python -m build
 
 publish:
-	python3 -m twine upload dist/*
+	poetry run twine upload dist/*
 
 twine-check:
-	python3 -m twine check dist/*
+	poetry run twine check dist/*
 
 lint:
-	python3 -m ruff check $(DIRS_WITH_CODE)
+	poetry run ruff check $(DIRS_WITH_CODE)
 
 unit-tests:
-	python3 -m pytest -n auto -ra tests/unit --cov=src/apify
+	poetry run pytest -n auto -ra tests/unit --cov=src/apify
 
 unit-tests-cov:
-	python3 -m pytest -n auto -ra tests/unit --cov=src/apify --cov-report=html
+	poetry run pytest -n auto -ra tests/unit --cov=src/apify --cov-report=html
 
 integration-tests:
-	python3 -m pytest -n $(INTEGRATION_TESTS_CONCURRENCY) -ra tests/integration
+	poetry run pytest -n $(INTEGRATION_TESTS_CONCURRENCY) -ra tests/integration
 
 type-check:
-	python3 -m mypy $(DIRS_WITH_CODE)
+	poetry run mypy $(DIRS_WITH_CODE)
 
 check-code: lint type-check unit-tests
 
 format:
-	python3 -m ruff check --fix $(DIRS_WITH_CODE)
-	python3 -m ruff format $(DIRS_WITH_CODE)
+	poetry run ruff check --fix $(DIRS_WITH_CODE)
+	poetry run ruff format $(DIRS_WITH_CODE)
 
 check-version-availability:
 	python3 scripts/check_version_availability.py

mypy.ini

@@ -14,6 +14,7 @@ warn_redundant_casts = True
 warn_return_any = True
 warn_unreachable = True
 warn_unused_ignores = True
+disable_error_code = attr-defined
 
 [mypy-scrapy.*]
 ignore_missing_imports = True