diff --git a/.github/workflows/run_checks.yaml b/.github/workflows/run_checks.yaml index 154f30fa..93dcff3b 100644 --- a/.github/workflows/run_checks.yaml +++ b/.github/workflows/run_checks.yaml @@ -21,7 +21,7 @@ jobs: integration_tests: if: github.event.pull_request.head.repo.owner.login == 'apify' name: Run integration tests - needs: [lint_and_type_checks, unit_tests] + needs: [lint_and_type_checks, unit_tests, unit_tests_poetry] uses: ./.github/workflows/integration_tests.yaml secrets: inherit diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index a5e3691d..826d85d5 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -8,8 +8,8 @@ jobs: name: Run unit tests strategy: matrix: - os: [ubuntu-latest, windows-latest] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + os: [ ubuntu-latest, macos-latest, windows-latest ] + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] runs-on: ${{ matrix.os }} steps: @@ -25,8 +25,19 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: make install-dev - - - name: Run unit tests - run: make unit-tests + - name: Install poetry + uses: abatilo/actions-poetry@v2 + - name: Setup a local virtual environment (if no poetry.toml file) + run: | + poetry config virtualenvs.create true --local + poetry config virtualenvs.in-project true --local + - uses: actions/cache@v3 + name: Define a cache for the virtual environment based on the dependencies lock file + with: + path: ./.venv + key: venv-${{ hashFiles('poetry.lock') }} + - name: Install the project dependencies + run: poetry install + - name: Run the automated tests (for example) + run: poetry run pytest tests/unit + continue-on-error: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7344a01b..22f5e594 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,45 +8,64 @@ It is recommended to set up a virtual environment while developing this package however, due to the many varied ways Python can be installed and virtual environments can be set up, this is left up to the developers to do themselves. -One recommended way is with the built-in `venv` module: +## Dependencies + +To install this package and its development dependencies, run: ```bash -python3 -m venv .venv -source .venv/bin/activate +make install-dev ``` -To improve on the experience, you can use [pyenv](https://github.com/pyenv/pyenv) to have an environment with a pinned Python version, -and [direnv](https://github.com/direnv/direnv) to automatically activate/deactivate the environment when you enter/exit the project folder. - -## Dependencies - -To install this package and its development dependencies, run `make install-dev`. - ## Code checking -To run all our code checking tools together, just run `make check-code`. +To run all our code checking tools together, just run: + +```bash +make check-code +``` ### Linting We use [ruff](https://docs.astral.sh/ruff/) for linting to to analyze the code for potential issues and enforce uniformed code style. See the `pyproject.toml` for its configuration. To run the linting, just run `make lint`. +```bash +make lint +``` + ### Formatting We use [ruff](https://docs.astral.sh/ruff/) for automated code formatting. It formats the code to follow uniformed code style and addresses auto-fixable linting issues. See the `pyproject.toml` for its configuration. To run -the formatting, just run `make format`. +the formatting, just run: + +```bash +make format +``` ### Type checking We use [mypy](https://mypy.readthedocs.io/en/stable/) for type checking. See the `mypy.ini` for its configuration. -To run the type checking, just run `make type-check`. +To run the type checking, just run: + +```bash +make type-check +``` ### Unit tests We use [pytest](https://docs.pytest.org/) as a testing framework with many plugins. See the `pyproject.toml` for -both its configuration and the list of installed plugins. To run unit tests execute `make unit-tests`. To run unit -tests with HTML coverage report execute `make unit-tests-cov`. +both its configuration and the list of installed plugins. To run unit tests execute: + +```bash +make unit-tests +``` + +with coverage report: + +```bash +make unit-tests-cov +``` ## Integration tests @@ -59,11 +78,14 @@ the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL to t ## Documentation -We use the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting the code. +We use the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for +documenting the code. We document every user-facing class or method, and enforce that using the flake8-docstrings library. -The documentation is then rendered from the docstrings in the code, using `pydoc-markdown` and some heavy post-processing, -and from Markdown documents in the `docs` folder in the `docs` branch, and then rendered using Docusaurus and published to GitHub pages. +The documentation is then rendered from the docstrings in the code, using `pydoc-markdown` and some heavy +post-processing, +and from Markdown documents in the `docs` folder in the `docs` branch, and then rendered using Docusaurus and published +to GitHub pages. ## Release process @@ -72,31 +94,38 @@ Publishing new versions to [PyPI](https://pypi.org/project/apify) happens automa On each commit to the `master` branch, a new beta release is published, taking the version number from `pyproject.toml` and automatically incrementing the beta version suffix by 1 from the last beta release published to PyPI. -A stable version is published when a new release is created using GitHub Releases, again taking the version number from `pyproject.toml`. +A stable version is published when a new release is created using GitHub Releases, again taking the version number +from `pyproject.toml`. The built package assets are automatically uploaded to the GitHub release. -If there is already a stable version with the same version number as in `pyproject.toml` published to PyPI, the publish process fails, +If there is already a stable version with the same version number as in `pyproject.toml` published to PyPI, the publish +process fails, so don't forget to update the version number before releasing a new version. The release process also fails when the released version is not described in `CHANGELOG.md`, so don't forget to describe the changes in the new version there. ### Beta release checklist -Beta release happens automatically after you merge a pull request or add a direct commit to the master branch. Before you do that check the following: +Beta release happens automatically after you merge a pull request or add a direct commit to the master branch. Before +you do that check the following: -- Make sure that in the [pyproject.toml](https://github.com/apify/apify-sdk-python/blob/master/pyproject.toml) a project version is set to the latest non-published version. -- Describe your changes to the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md) in the section with the latest non-published version. +- Make sure that in the [pyproject.toml](https://github.com/apify/apify-sdk-python/blob/master/pyproject.toml) a project + version is set to the latest non-published version. +- Describe your changes to the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md) in the + section with the latest non-published version. ### Production release checklist Production release happens after the GitHub release is created. Before you do that check the following: - Make sure that the beta release with the latest commit is successfully deployed. -- Make sure that all the changes that happened from the last production release are described in the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md). +- Make sure that all the changes that happened from the last production release are described in + the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md). - When drafting a new GitHub release: - Create a new tag in the format of `v1.2.3` targeting the master branch. - Fill in the release title in the format of `1.2.3`. - - Copy the changes from the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md) and paste them into the release description. + - Copy the changes from the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md) and + paste them into the release description. - Check the "Set as the latest release" option. ## Maintanance @@ -108,29 +137,29 @@ Production release happens after the GitHub release is created. Before you do th ### Adding support for a new Python version 1) Firstly, ensure that the package ( - [apify-sdk-python](https://github.com/apify/apify-sdk-python), - [apify-client-python](https://github.com/apify/apify-client-python), - [apify-shared-python](https://github.com/apify/apify-shared-python) -) is compatible with the new Python version. Both in our code base and -the dependencies we use. Then, release a new version of the package. + [apify-sdk-python](https://github.com/apify/apify-sdk-python), + [apify-client-python](https://github.com/apify/apify-client-python), + [apify-shared-python](https://github.com/apify/apify-shared-python) + ) is compatible with the new Python version. Both in our code base and + the dependencies we use. Then, release a new version of the package. - For inspiration, see the PR - [apify/apify-sdk-python#121](https://github.com/apify/apify-sdk-python/pull/121), - where support for Python 3.12 was added to the Apify Python SDK. + [apify/apify-sdk-python#121](https://github.com/apify/apify-sdk-python/pull/121), + where support for Python 3.12 was added to the Apify Python SDK. 2) Next, build and publish the new versions of Python base Docker images. - For inspiration, see the PR - [apify/apify-actor-docker#112](https://github.com/apify/apify-actor-docker/pull/112), - where support for Python 3.12 was added. + [apify/apify-actor-docker#112](https://github.com/apify/apify-actor-docker/pull/112), + where support for Python 3.12 was added. - Apify base Docker images are built using GitHub Actions, accessible at - [apify/apify-actor-docker/actions](https://github.com/apify/apify-actor-docker/actions). + [apify/apify-actor-docker/actions](https://github.com/apify/apify-actor-docker/actions). 3) Integrate the new Python version into the CI/CD workflows -of existing Python projects ( - [apify-sdk-python](https://github.com/apify/apify-sdk-python), - [apify-client-python](https://github.com/apify/apify-client-python), - [apify-shared-python](https://github.com/apify/apify-shared-python), - [actor-templates](https://github.com/apify/actor-templates) -). + of existing Python projects ( + [apify-sdk-python](https://github.com/apify/apify-sdk-python), + [apify-client-python](https://github.com/apify/apify-client-python), + [apify-shared-python](https://github.com/apify/apify-shared-python), + [actor-templates](https://github.com/apify/actor-templates) + ). - For inspiration, see the PR - [apify/apify-sdk-python#124](https://github.com/apify/apify-sdk-python/pull/124), - where support for Python 3.12 was added to the CI/CD of the Apify Python SDK. + [apify/apify-sdk-python#124](https://github.com/apify/apify-sdk-python/pull/124), + where support for Python 3.12 was added to the CI/CD of the Apify Python SDK. diff --git a/Makefile b/Makefile index 4fb00681..c92fec7f 100644 --- a/Makefile +++ b/Makefile @@ -9,39 +9,41 @@ clean: rm -rf build dist .mypy_cache .pytest_cache src/*.egg-info __pycache__ install-dev: - python3 -m pip install --upgrade pip - pip install --no-cache-dir -e ".[dev,scrapy]" - pre-commit install + pip install poetry + poetry config virtualenvs.create true --local + poetry config virtualenvs.in-project true --local + poetry install --all-extras + poetry run pre-commit install build: - python3 -m build + poetry run python -m build publish: - python3 -m twine upload dist/* + poetry run twine upload dist/* twine-check: - python3 -m twine check dist/* + poetry run twine check dist/* lint: - python3 -m ruff check $(DIRS_WITH_CODE) + poetry run ruff check $(DIRS_WITH_CODE) unit-tests: - python3 -m pytest -n auto -ra tests/unit --cov=src/apify + poetry run pytest -n auto -ra tests/unit --cov=src/apify unit-tests-cov: - python3 -m pytest -n auto -ra tests/unit --cov=src/apify --cov-report=html + poetry run pytest -n auto -ra tests/unit --cov=src/apify --cov-report=html integration-tests: - python3 -m pytest -n $(INTEGRATION_TESTS_CONCURRENCY) -ra tests/integration + poetry run pytest -n $(INTEGRATION_TESTS_CONCURRENCY) -ra tests/integration type-check: - python3 -m mypy $(DIRS_WITH_CODE) + poetry run mypy $(DIRS_WITH_CODE) check-code: lint type-check unit-tests format: - python3 -m ruff check --fix $(DIRS_WITH_CODE) - python3 -m ruff format $(DIRS_WITH_CODE) + poetry run ruff check --fix $(DIRS_WITH_CODE) + poetry run ruff format $(DIRS_WITH_CODE) check-version-availability: python3 scripts/check_version_availability.py diff --git a/mypy.ini b/mypy.ini index df472b2f..775f5838 100644 --- a/mypy.ini +++ b/mypy.ini @@ -14,6 +14,7 @@ warn_redundant_casts = True warn_return_any = True warn_unreachable = True warn_unused_ignores = True +disable_error_code = attr-defined [mypy-scrapy.*] ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index 43e00000..ef7539c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,11 @@ -[project] +[tool.poetry] name = "apify" version = "1.5.6" description = "Apify SDK for Python" readme = "README.md" -license = { text = "Apache Software License" } -authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }] +license = "Apache Software License" +authors = ["Apify Technologies s.r.o. "] keywords = ["apify", "sdk", "actor", "scraping", "automation"] - classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", @@ -19,52 +18,47 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Topic :: Software Development :: Libraries", ] +[tool.poetry.dependencies] +python = ">=3.8,<4.0" +apify-client = "*" +apify-shared = "*" +aiofiles = "^22.1.0" +aioshutil = "^1.0" +colorama = "^0.4.6" +cryptography = "^39.0.0" +httpx = "^0.24.1" +psutil = "^5.9.5" +pyee = "^11.0.1" +sortedcollections = "^2.0.1" +typing-extensions = "^4.1.0" +websockets = "^10.1" -requires-python = ">=3.8" -# We use inclusive ordered comparison clause for non-Apify packages intentionally in order to enhance the Apify SDK's -# compatibility with a wide range of external packages. This decision was discussed in detail in the following PR: -# https://github.com/apify/apify-sdk-python/pull/154 -dependencies = [ - "apify-client ~= 1.6.2", - "apify-shared ~= 1.1.1", - "aiofiles >= 22.1.0", - "aioshutil >= 1.0", - "colorama >= 0.4.6", - "cryptography >= 39.0.0", - "httpx >= 0.24.1", - "psutil >= 5.9.5", - "pyee >= 11.0.1", - "sortedcollections >= 2.0.1", - "typing-extensions >= 4.1.0", - "websockets >= 10.1", -] +[tool.poetry.dev-dependencies] +# Include all optional and scrapy dependencies from the provided file +build = "^1.0.3" +filelock = "^3.12.4" +mypy = "^1.7.1" +pre-commit = "^3.4.0" +pydoc-markdown = "^4.8.2" +pytest = "^7.4.2" +pytest-asyncio = "^0.21.0" +pytest-cov = "^4.1.0" +pytest-only = "^2.0.0" +pytest-timeout = "^2.2.0" +pytest-xdist = "^3.3.1" +respx = "^0.20.1" +ruff = "^0.1.13" +twine = "^4.0.2" +types-aiofiles = "^23.2.0.0" +types-colorama = "^0.4.15.12" +types-psutil = "^5.9.5.17" +scrapy = { version = "^2.11.0", extras = ["scrapy"] } -[project.optional-dependencies] -dev = [ - "build ~= 1.0.3", - "filelock ~= 3.12.4", - "mypy ~= 1.7.1", - "pre-commit ~= 3.4.0", - "pydoc-markdown ~= 4.8.2", - "pytest ~= 7.4.2", - "pytest-asyncio ~= 0.21.0", - "pytest-cov ~= 4.1.0", - "pytest-only ~= 2.0.0", - "pytest-timeout ~= 2.2.0", - "pytest-xdist ~= 3.3.1", - "respx ~= 0.20.1", - "ruff ~= 0.1.13", - "twine ~= 4.0.2", - "types-aiofiles ~= 23.2.0.0", - "types-colorama ~= 0.4.15.12", - "types-psutil ~= 5.9.5.17", -] -scrapy = [ - "scrapy >= 2.11.0", -] +[tool.poetry.extras] +scrapy = ["scrapy"] -[project.urls] +[tool.poetry.urls] "Homepage" = "https://docs.apify.com/sdk/python/" "Documentation" = "https://docs.apify.com/sdk/python/" "Source" = "https://github.com/apify/apify-sdk-python" @@ -73,45 +67,40 @@ scrapy = [ "Apify Homepage" = "https://apify.com" [build-system] -requires = ["setuptools>=64.0.0", "wheel"] -build-backend = "setuptools.build_meta" +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" -[tool.setuptools.packages.find] -where = ["src"] -include = ["apify*"] -[tool.setuptools.package-data] -apify = ["py.typed"] [tool.ruff] line-length = 150 select = ["ALL"] ignore = [ - "ANN401", # Dynamically typed expressions (typing.Any) are disallowed in {filename} - "BLE001", # Do not catch blind exception - "C901", # `{name}` is too complex - "COM812", # This rule may cause conflicts when used with the formatter - "D100", # Missing docstring in public module - "D104", # Missing docstring in public package - "EM", # flake8-errmsg - "G004", # Logging statement uses f-string - "ISC001", # This rule may cause conflicts when used with the formatter - "FIX", # flake8-fixme - "PGH003", # Use specific rule codes when ignoring type issues + "ANN401", # Dynamically typed expressions (typing.Any) are disallowed in {filename} + "BLE001", # Do not catch blind exception + "C901", # `{name}` is too complex + "COM812", # This rule may cause conflicts when used with the formatter + "D100", # Missing docstring in public module + "D104", # Missing docstring in public package + "EM", # flake8-errmsg + "G004", # Logging statement uses f-string + "ISC001", # This rule may cause conflicts when used with the formatter + "FIX", # flake8-fixme + "PGH003", # Use specific rule codes when ignoring type issues "PLR0911", # Too many return statements "PLR0913", # Too many arguments in function definition "PLR0915", # Too many statements - "PTH", # flake8-use-pathlib - "PYI034", # `__aenter__` methods in classes like `{name}` usually return `self` at runtime - "PYI036", # The second argument in `__aexit__` should be annotated with `object` or `BaseException | None` - "S102", # Use of `exec` detected - "S105", # Possible hardcoded password assigned to - "S106", # Possible hardcoded password assigned to argument: "{name}" - "S301", # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue - "S303", # Use of insecure MD2, MD4, MD5, or SHA1 hash function - "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes - "TD002", # Missing author in TODO; try: `# TODO(): ...` or `# TODO @: ... - "TRY003", # Avoid specifying long messages outside the exception class + "PTH", # flake8-use-pathlib + "PYI034", # `__aenter__` methods in classes like `{name}` usually return `self` at runtime + "PYI036", # The second argument in `__aexit__` should be annotated with `object` or `BaseException | None` + "S102", # Use of `exec` detected + "S105", # Possible hardcoded password assigned to + "S106", # Possible hardcoded password assigned to argument: "{name}" + "S301", # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue + "S303", # Use of insecure MD2, MD4, MD5, or SHA1 hash function + "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes + "TD002", # Missing author in TODO; try: `# TODO(): ...` or `# TODO @: ... + "TRY003", # Avoid specifying long messages outside the exception class # TODO: Remove this once the following issue is fixed # https://github.com/apify/apify-sdk-python/issues/150 @@ -127,20 +116,20 @@ indent-style = "space" "F401", # Unused imports ] "**/{scripts}/*" = [ - "D", # Everything from the pydocstyle - "INP001", # File {filename} is part of an implicit namespace package, add an __init__.py + "D", # Everything from the pydocstyle + "INP001", # File {filename} is part of an implicit namespace package, add an __init__.py "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable - "T20", # flake8-print + "T20", # flake8-print ] "**/{tests}/*" = [ - "D", # Everything from the pydocstyle - "INP001", # File {filename} is part of an implicit namespace package, add an __init__.py - "PT011", # `pytest.raises({ExceptionType})` is too broad, set the `match` parameter or use a more specific exception + "D", # Everything from the pydocstyle + "INP001", # File {filename} is part of an implicit namespace package, add an __init__.py + "PT011", # `pytest.raises({ExceptionType})` is too broad, set the `match` parameter or use a more specific exception "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable - "S101", # Use of assert detected - "T20", # flake8-print - "TID252", # Relative imports from parent modules are banned - "TRY301", # Abstract `raise` to an inner function + "S101", # Use of assert detected + "T20", # flake8-print + "TID252", # Relative imports from parent modules are banned + "TRY301", # Abstract `raise` to an inner function ] [tool.ruff.lint.flake8-quotes]