From fecf55ae3c38eec565fc91f48dcfc070a31d7eba Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Thu, 15 Feb 2024 18:49:13 +0200
Subject: [PATCH 01/18] poetry first try.

---
 .github/workflows/unit_tests_poetry.yml |  38 +++++++
 pyproject.toml                          | 127 ++++++++++++++----------
 2 files changed, 112 insertions(+), 53 deletions(-)
 create mode 100644 .github/workflows/unit_tests_poetry.yml

diff --git a/.github/workflows/unit_tests_poetry.yml b/.github/workflows/unit_tests_poetry.yml
new file mode 100644
index 00000000..5cccfacd
--- /dev/null
+++ b/.github/workflows/unit_tests_poetry.yml
@@ -0,0 +1,38 @@
+name: Unit tests
+
+on:
+  [ pull_request, push]
+
+jobs:
+    unit-test:
+        runs-on: ${{ matrix.os }}
+        strategy:
+            matrix:
+                os: [ ubuntu-latest, macos-latest, windows-latest ]
+                python-version: [ '3.9', '3.10', '3.11', '3.12' ]
+
+        steps:
+            -   uses: actions/checkout@v3
+
+            -   name: Install Python
+                uses: actions/setup-python@v4
+                with:
+                    python-version: ${{ matrix.python-version }}
+
+            -   name: Install poetry
+                uses: abatilo/actions-poetry@v2
+            -   name: Setup a local virtual environment (if no poetry.toml file)
+                run: |
+                    poetry config virtualenvs.create true --local
+                    poetry config virtualenvs.in-project true --local
+            -   uses: actions/cache@v3
+                name: Define a cache for the virtual environment based on the dependencies lock file
+                with:
+                    path: ./.venv
+                    key: venv-${{ hashFiles('poetry.lock') }}
+            -   name: Install the project dependencies
+                run: poetry install
+            -   name: Run the automated tests (for example)
+                run: poetry run pytest
+        continue-on-error: true
+
diff --git a/pyproject.toml b/pyproject.toml
index 43e00000..4a546547 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,12 +1,11 @@
-[project]
+[tool.poetry]
 name = "apify"
 version = "1.5.6"
 description = "Apify SDK for Python"
 readme = "README.md"
-license = { text = "Apache Software License" }
-authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
+license = "Apache Software License"
+authors = ["Apify Technologies s.r.o. <support@apify.com>"]
 keywords = ["apify", "sdk", "actor", "scraping", "automation"]
-
 classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Intended Audience :: Developers",
@@ -19,52 +18,79 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
     "Topic :: Software Development :: Libraries",
 ]
+[tool.poetry.dependencies]
+python = ">=3.8,<4.0"
+#required = [
+#    "apify-client ~= 1.6.2",
+#    "apify-shared ~= 1.1.1",
+#    "aiofiles >= 22.1.0",
+#    "aioshutil >= 1.0",
+#    "colorama >= 0.4.6",
+#    "cryptography >= 39.0.0",
+#    "httpx >= 0.24.1",
+#    "psutil >= 5.9.5",
+#    "pyee >= 11.0.1",
+#    "sortedcollections >= 2.0.1",
+#    "typing-extensions >= 4.1.0",
+#    "websockets >= 10.1",
+#]
+apify-client = "*"
+apify-shared = "*"
+aiofiles = "^22.1.0"
+aioshutil = "^1.0"
+colorama = "^0.4.6"
+cryptography = "^39.0.0"
+httpx = "^0.24.1"
+psutil = "^5.9.5"
+pyee = "^11.0.1"
+sortedcollections = "^2.0.1"
+typing-extensions = "^4.1.0"
+websockets = "^10.1"
 
-requires-python = ">=3.8"
 
-# We use inclusive ordered comparison clause for non-Apify packages intentionally in order to enhance the Apify SDK's
-# compatibility with a wide range of external packages. This decision was discussed in detail in the following PR:
-# https://github.com/apify/apify-sdk-python/pull/154
-dependencies = [
-    "apify-client ~= 1.6.2",
-    "apify-shared ~= 1.1.1",
-    "aiofiles >= 22.1.0",
-    "aioshutil >= 1.0",
-    "colorama >= 0.4.6",
-    "cryptography >= 39.0.0",
-    "httpx >= 0.24.1",
-    "psutil >= 5.9.5",
-    "pyee >= 11.0.1",
-    "sortedcollections >= 2.0.1",
-    "typing-extensions >= 4.1.0",
-    "websockets >= 10.1",
-]
+[tool.poetry.dev-dependencies]
+# Include all optional and scrapy dependencies from the provided file
+build = "^1.0.3"
+filelock = "^3.12.4"
+mypy = "^1.7.1"
+pre-commit = "^3.4.0"
+pydoc-markdown = "^4.8.2"
+pytest = "^7.4.2"
+pytest-asyncio = "^0.21.0"
+pytest-cov = "^4.1.0"
+pytest-only = "^2.0.0"
+pytest-timeout = "^2.2.0"
+pytest-xdist = "^3.3.1"
+respx = "^0.20.1"
+ruff = "^0.1.13"
+twine = "^4.0.2"
+types-aiofiles = "^23.2.0.0"
+types-colorama = "^0.4.15.12"
+types-psutil = "^5.9.5.17"
+scrapy = ">=2.11.0"
 
-[project.optional-dependencies]
-dev = [
-    "build ~= 1.0.3",
-    "filelock ~= 3.12.4",
-    "mypy ~= 1.7.1",
-    "pre-commit ~= 3.4.0",
-    "pydoc-markdown ~= 4.8.2",
-    "pytest ~= 7.4.2",
-    "pytest-asyncio ~= 0.21.0",
-    "pytest-cov ~= 4.1.0",
-    "pytest-only ~= 2.0.0",
-    "pytest-timeout ~= 2.2.0",
-    "pytest-xdist ~= 3.3.1",
-    "respx ~= 0.20.1",
-    "ruff ~= 0.1.13",
-    "twine ~= 4.0.2",
-    "types-aiofiles ~= 23.2.0.0",
-    "types-colorama ~= 0.4.15.12",
-    "types-psutil ~= 5.9.5.17",
-]
-scrapy = [
-    "scrapy >= 2.11.0",
-]
+#dev = [
+#    "build ~= 1.0.3",
+#    "filelock ~= 3.12.4",
+#    "mypy ~= 1.7.1",
+#    "pre-commit ~= 3.4.0",
+#    "pydoc-markdown ~= 4.8.2",
+#    "pytest ~= 7.4.2",
+#    "pytest-asyncio ~= 0.21.0",
+#    "pytest-cov ~= 4.1.0",
+#    "pytest-only ~= 2.0.0",
+#    "pytest-timeout ~= 2.2.0",
+#    "pytest-xdist ~= 3.3.1",
+#    "respx ~= 0.20.1",
+#    "ruff ~= 0.1.13",
+#    "twine ~= 4.0.2",
+#    "types-aiofiles ~= 23.2.0.0",
+#    "types-colorama ~= 0.4.15.12",
+#    "types-psutil ~= 5.9.5.17",
+#    "scrapy >= 2.11.0"
+#]
 
-[project.urls]
+[tool.poetry.urls]
 "Homepage" = "https://docs.apify.com/sdk/python/"
 "Documentation" = "https://docs.apify.com/sdk/python/"
 "Source" = "https://github.com/apify/apify-sdk-python"
@@ -73,15 +99,10 @@ scrapy = [
 "Apify Homepage" = "https://apify.com"
 
 [build-system]
-requires = ["setuptools>=64.0.0", "wheel"]
-build-backend = "setuptools.build_meta"
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
 
-[tool.setuptools.packages.find]
-where = ["src"]
-include = ["apify*"]
 
-[tool.setuptools.package-data]
-apify = ["py.typed"]
 
 [tool.ruff]
 line-length = 150

From ef57df08dddcf39a3a0efdb9c967044922933431 Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Thu, 15 Feb 2024 18:52:54 +0200
Subject: [PATCH 02/18] pytest execute unit only.

---
 .github/workflows/unit_tests_poetry.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/unit_tests_poetry.yml b/.github/workflows/unit_tests_poetry.yml
index 5cccfacd..75ad16a2 100644
--- a/.github/workflows/unit_tests_poetry.yml
+++ b/.github/workflows/unit_tests_poetry.yml
@@ -33,6 +33,6 @@ jobs:
             -   name: Install the project dependencies
                 run: poetry install
             -   name: Run the automated tests (for example)
-                run: poetry run pytest
+                run: poetry run pytest test/unit
         continue-on-error: true
 

From 972812212e1963b5a8023c1ff655711c1f6ccdf6 Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Thu, 15 Feb 2024 18:56:27 +0200
Subject: [PATCH 03/18] fixed typo.

---
 .github/workflows/unit_tests_poetry.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/unit_tests_poetry.yml b/.github/workflows/unit_tests_poetry.yml
index 75ad16a2..da4700fe 100644
--- a/.github/workflows/unit_tests_poetry.yml
+++ b/.github/workflows/unit_tests_poetry.yml
@@ -33,6 +33,6 @@ jobs:
             -   name: Install the project dependencies
                 run: poetry install
             -   name: Run the automated tests (for example)
-                run: poetry run pytest test/unit
+                run: poetry run pytest tests/unit
         continue-on-error: true
 

From 6b25152be087bc17169fff2550d199817e2c6035 Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Thu, 15 Feb 2024 19:06:24 +0200
Subject: [PATCH 04/18] update make dev with poetry.

---
 Makefile | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 4fb00681..46f675d7 100644
--- a/Makefile
+++ b/Makefile
@@ -9,9 +9,8 @@ clean:
 	rm -rf build dist .mypy_cache .pytest_cache src/*.egg-info __pycache__
 
 install-dev:
-	python3 -m pip install --upgrade pip
-	pip install --no-cache-dir -e ".[dev,scrapy]"
-	pre-commit install
+	poetry install
+	poetry run pre-commit install
 
 build:
 	python3 -m build

From 47ae4a3f7be07ae16d745a56d2fc48366853e4e4 Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Thu, 15 Feb 2024 19:09:35 +0200
Subject: [PATCH 05/18] install poetry for poetry commands.

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 46f675d7..511c329a 100644
--- a/Makefile
+++ b/Makefile
@@ -9,6 +9,7 @@ clean:
 	rm -rf build dist .mypy_cache .pytest_cache src/*.egg-info __pycache__
 
 install-dev:
+	pip install poetry
 	poetry install
 	poetry run pre-commit install
 

From 9d3cf89e3b410631bb09e82d21a5d9e6031ca0bd Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Thu, 15 Feb 2024 19:15:41 +0200
Subject: [PATCH 06/18] replace python runs with poetry run.

---
 Makefile | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/Makefile b/Makefile
index 511c329a..e0333ef7 100644
--- a/Makefile
+++ b/Makefile
@@ -14,34 +14,34 @@ install-dev:
 	poetry run pre-commit install
 
 build:
-	python3 -m build
+	poetry run build
 
 publish:
-	python3 -m twine upload dist/*
+	poetry run twine upload dist/*
 
 twine-check:
-	python3 -m twine check dist/*
+	poetry run twine check dist/*
 
 lint:
-	python3 -m ruff check $(DIRS_WITH_CODE)
+	poetry run ruff check $(DIRS_WITH_CODE)
 
 unit-tests:
-	python3 -m pytest -n auto -ra tests/unit --cov=src/apify
+	poetry run pytest -n auto -ra tests/unit --cov=src/apify
 
 unit-tests-cov:
-	python3 -m pytest -n auto -ra tests/unit --cov=src/apify --cov-report=html
+	poetry run pytest -n auto -ra tests/unit --cov=src/apify --cov-report=html
 
 integration-tests:
-	python3 -m pytest -n $(INTEGRATION_TESTS_CONCURRENCY) -ra tests/integration
+	poetry run pytest -n $(INTEGRATION_TESTS_CONCURRENCY) -ra tests/integration
 
 type-check:
-	python3 -m mypy $(DIRS_WITH_CODE)
+	poetry run mypy $(DIRS_WITH_CODE)
 
 check-code: lint type-check unit-tests
 
 format:
-	python3 -m ruff check --fix $(DIRS_WITH_CODE)
-	python3 -m ruff format $(DIRS_WITH_CODE)
+	poetry run ruff check --fix $(DIRS_WITH_CODE)
+	poetry run ruff format $(DIRS_WITH_CODE)
 
 check-version-availability:
 	python3 scripts/check_version_availability.py

From 5f3ad55ed749504552616edd88c2b98353f1cb84 Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Thu, 15 Feb 2024 19:39:14 +0200
Subject: [PATCH 07/18] add poetry config to makefile install dev.

---
 Makefile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Makefile b/Makefile
index e0333ef7..4a99aef9 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,10 @@ clean:
 	rm -rf build dist .mypy_cache .pytest_cache src/*.egg-info __pycache__
 
 install-dev:
+
 	pip install poetry
+	poetry config virtualenvs.create true --local
+	poetry config virtualenvs.in-project true --local
 	poetry install
 	poetry run pre-commit install
 

From 2bd38aa55aad2f69d35eb6b28ecd22ef6e9a706e Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Thu, 15 Feb 2024 19:53:21 +0200
Subject: [PATCH 08/18] update CONTRIBUTING.md.

Signed-off-by: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
---
 CONTRIBUTING.md | 115 ++++++++++++++++++++++++++++++------------------
 1 file changed, 72 insertions(+), 43 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7344a01b..22f5e594 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -8,45 +8,64 @@ It is recommended to set up a virtual environment while developing this package
 however, due to the many varied ways Python can be installed and virtual environments can be set up,
 this is left up to the developers to do themselves.
 
-One recommended way is with the built-in `venv` module:
+## Dependencies
+
+To install this package and its development dependencies, run:
 
 ```bash
-python3 -m venv .venv
-source .venv/bin/activate
+make install-dev
 ```
 
-To improve on the experience, you can use [pyenv](https://github.com/pyenv/pyenv) to have an environment with a pinned Python version,
-and [direnv](https://github.com/direnv/direnv) to automatically activate/deactivate the environment when you enter/exit the project folder.
-
-## Dependencies
-
-To install this package and its development dependencies, run `make install-dev`.
-
 ## Code checking
 
-To run all our code checking tools together, just run `make check-code`.
+To run all our code checking tools together, just run:
+
+```bash
+make check-code
+```
 
 ### Linting
 
 We use [ruff](https://docs.astral.sh/ruff/) for linting to to analyze the code for potential issues and enforce
 uniformed code style. See the `pyproject.toml` for its configuration. To run the linting, just run `make lint`.
 
+```bash
+make lint
+```
+
 ### Formatting
 
 We use [ruff](https://docs.astral.sh/ruff/) for automated code formatting. It formats the code to follow uniformed
 code style and addresses auto-fixable linting issues. See the `pyproject.toml` for its configuration. To run
-the formatting, just run `make format`.
+the formatting, just run:
+
+```bash
+make format
+```
 
 ### Type checking
 
 We use [mypy](https://mypy.readthedocs.io/en/stable/) for type checking. See the `mypy.ini` for its configuration.
-To run the type checking, just run `make type-check`.
+To run the type checking, just run:
+
+```bash
+make type-check
+```
 
 ### Unit tests
 
 We use [pytest](https://docs.pytest.org/) as a testing framework with many plugins. See the `pyproject.toml` for
-both its configuration and the list of installed plugins. To run unit tests execute `make unit-tests`. To run unit
-tests with HTML coverage report execute `make unit-tests-cov`.
+both its configuration and the list of installed plugins. To run unit tests execute:
+
+```bash
+make unit-tests
+```
+
+with coverage report:
+
+```bash
+make unit-tests-cov
+```
 
 ## Integration tests
 
@@ -59,11 +78,14 @@ the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL to t
 
 ## Documentation
 
-We use the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting the code.
+We use the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for
+documenting the code.
 We document every user-facing class or method, and enforce that using the flake8-docstrings library.
 
-The documentation is then rendered from the docstrings in the code, using `pydoc-markdown` and some heavy post-processing,
-and from Markdown documents in the `docs` folder in the `docs` branch, and then rendered using Docusaurus and published to GitHub pages.
+The documentation is then rendered from the docstrings in the code, using `pydoc-markdown` and some heavy
+post-processing,
+and from Markdown documents in the `docs` folder in the `docs` branch, and then rendered using Docusaurus and published
+to GitHub pages.
 
 ## Release process
 
@@ -72,31 +94,38 @@ Publishing new versions to [PyPI](https://pypi.org/project/apify) happens automa
 On each commit to the `master` branch, a new beta release is published, taking the version number from `pyproject.toml`
 and automatically incrementing the beta version suffix by 1 from the last beta release published to PyPI.
 
-A stable version is published when a new release is created using GitHub Releases, again taking the version number from `pyproject.toml`.
+A stable version is published when a new release is created using GitHub Releases, again taking the version number
+from `pyproject.toml`.
 The built package assets are automatically uploaded to the GitHub release.
 
-If there is already a stable version with the same version number as in `pyproject.toml` published to PyPI, the publish process fails,
+If there is already a stable version with the same version number as in `pyproject.toml` published to PyPI, the publish
+process fails,
 so don't forget to update the version number before releasing a new version.
 The release process also fails when the released version is not described in `CHANGELOG.md`,
 so don't forget to describe the changes in the new version there.
 
 ### Beta release checklist
 
-Beta release happens automatically after you merge a pull request or add a direct commit to the master branch. Before you do that check the following:
+Beta release happens automatically after you merge a pull request or add a direct commit to the master branch. Before
+you do that check the following:
 
-- Make sure that in the [pyproject.toml](https://github.com/apify/apify-sdk-python/blob/master/pyproject.toml) a project version is set to the latest non-published version.
-- Describe your changes to the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md) in the section with the latest non-published version.
+- Make sure that in the [pyproject.toml](https://github.com/apify/apify-sdk-python/blob/master/pyproject.toml) a project
+  version is set to the latest non-published version.
+- Describe your changes to the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md) in the
+  section with the latest non-published version.
 
 ### Production release checklist
 
 Production release happens after the GitHub release is created. Before you do that check the following:
 
 - Make sure that the beta release with the latest commit is successfully deployed.
-- Make sure that all the changes that happened from the last production release are described in the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md).
+- Make sure that all the changes that happened from the last production release are described in
+  the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md).
 - When drafting a new GitHub release:
     - Create a new tag in the format of `v1.2.3` targeting the master branch.
     - Fill in the release title in the format of `1.2.3`.
-    - Copy the changes from the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md) and paste them into the release description.
+    - Copy the changes from the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md) and
+      paste them into the release description.
     - Check the "Set as the latest release" option.
 
 ## Maintanance
@@ -108,29 +137,29 @@ Production release happens after the GitHub release is created. Before you do th
 ### Adding support for a new Python version
 
 1) Firstly, ensure that the package (
-    [apify-sdk-python](https://github.com/apify/apify-sdk-python),
-    [apify-client-python](https://github.com/apify/apify-client-python),
-    [apify-shared-python](https://github.com/apify/apify-shared-python)
-) is compatible with the new Python version. Both in our code base and
-the dependencies we use. Then, release a new version of the package.
+   [apify-sdk-python](https://github.com/apify/apify-sdk-python),
+   [apify-client-python](https://github.com/apify/apify-client-python),
+   [apify-shared-python](https://github.com/apify/apify-shared-python)
+   ) is compatible with the new Python version. Both in our code base and
+   the dependencies we use. Then, release a new version of the package.
     - For inspiration, see the PR
-    [apify/apify-sdk-python#121](https://github.com/apify/apify-sdk-python/pull/121),
-    where support for Python 3.12 was added to the Apify Python SDK.
+      [apify/apify-sdk-python#121](https://github.com/apify/apify-sdk-python/pull/121),
+      where support for Python 3.12 was added to the Apify Python SDK.
 
 2) Next, build and publish the new versions of Python base Docker images.
     - For inspiration, see the PR
-    [apify/apify-actor-docker#112](https://github.com/apify/apify-actor-docker/pull/112),
-    where support for Python 3.12 was added.
+      [apify/apify-actor-docker#112](https://github.com/apify/apify-actor-docker/pull/112),
+      where support for Python 3.12 was added.
     - Apify base Docker images are built using GitHub Actions, accessible at
-    [apify/apify-actor-docker/actions](https://github.com/apify/apify-actor-docker/actions).
+      [apify/apify-actor-docker/actions](https://github.com/apify/apify-actor-docker/actions).
 
 3) Integrate the new Python version into the CI/CD workflows
-of existing Python projects (
-    [apify-sdk-python](https://github.com/apify/apify-sdk-python),
-    [apify-client-python](https://github.com/apify/apify-client-python),
-    [apify-shared-python](https://github.com/apify/apify-shared-python),
-    [actor-templates](https://github.com/apify/actor-templates)
-).
+   of existing Python projects (
+   [apify-sdk-python](https://github.com/apify/apify-sdk-python),
+   [apify-client-python](https://github.com/apify/apify-client-python),
+   [apify-shared-python](https://github.com/apify/apify-shared-python),
+   [actor-templates](https://github.com/apify/actor-templates)
+   ).
     - For inspiration, see the PR
-    [apify/apify-sdk-python#124](https://github.com/apify/apify-sdk-python/pull/124),
-    where support for Python 3.12 was added to the CI/CD of the Apify Python SDK.
+      [apify/apify-sdk-python#124](https://github.com/apify/apify-sdk-python/pull/124),
+      where support for Python 3.12 was added to the CI/CD of the Apify Python SDK.

From 389bae968bd64ae4164629af2ef2d783e65cacc9 Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Fri, 16 Feb 2024 10:43:59 +0200
Subject: [PATCH 09/18] supress mypy attr not define, caused by socket lazy
 import.

Signed-off-by: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
---
 mypy.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mypy.ini b/mypy.ini
index df472b2f..775f5838 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -14,6 +14,7 @@ warn_redundant_casts = True
 warn_return_any = True
 warn_unreachable = True
 warn_unused_ignores = True
+disable_error_code = attr-defined
 
 [mypy-scrapy.*]
 ignore_missing_imports = True

From fee9efce2651a8c4313828aed30b1b22f98bca4a Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Fri, 16 Feb 2024 10:52:01 +0200
Subject: [PATCH 10/18] update run_checks.yaml workflow with poetry unittests.

Signed-off-by: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
---
 .github/workflows/run_checks.yaml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run_checks.yaml b/.github/workflows/run_checks.yaml
index 154f30fa..9128f2eb 100644
--- a/.github/workflows/run_checks.yaml
+++ b/.github/workflows/run_checks.yaml
@@ -17,11 +17,16 @@ jobs:
     needs: [lint_and_type_checks]
     uses: ./.github/workflows/unit_tests.yaml
 
+  unit_tests_poetry:
+    name: Run unit tests
+    needs: [ lint_and_type_checks ]
+    uses: ./.github/workflows/unit_tests_poetry.yml
+
   # If the PR comes from the main repo, run integration tests directly
   integration_tests:
     if: github.event.pull_request.head.repo.owner.login == 'apify'
     name: Run integration tests
-    needs: [lint_and_type_checks, unit_tests]
+    needs: [lint_and_type_checks, unit_tests, unit_tests_poetry]
     uses: ./.github/workflows/integration_tests.yaml
     secrets: inherit
 

From bc50cefd5143f59c14edc2250292414b223b7f9e Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Fri, 16 Feb 2024 10:57:41 +0200
Subject: [PATCH 11/18] refactored pyproject.toml.

Signed-off-by: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
---
 pyproject.toml | 103 +++++++++++++++++--------------------------------
 1 file changed, 36 insertions(+), 67 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 4a546547..581bb84a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -20,20 +20,6 @@ classifiers = [
 ]
 [tool.poetry.dependencies]
 python = ">=3.8,<4.0"
-#required = [
-#    "apify-client ~= 1.6.2",
-#    "apify-shared ~= 1.1.1",
-#    "aiofiles >= 22.1.0",
-#    "aioshutil >= 1.0",
-#    "colorama >= 0.4.6",
-#    "cryptography >= 39.0.0",
-#    "httpx >= 0.24.1",
-#    "psutil >= 5.9.5",
-#    "pyee >= 11.0.1",
-#    "sortedcollections >= 2.0.1",
-#    "typing-extensions >= 4.1.0",
-#    "websockets >= 10.1",
-#]
 apify-client = "*"
 apify-shared = "*"
 aiofiles = "^22.1.0"
@@ -67,28 +53,11 @@ twine = "^4.0.2"
 types-aiofiles = "^23.2.0.0"
 types-colorama = "^0.4.15.12"
 types-psutil = "^5.9.5.17"
-scrapy = ">=2.11.0"
+scrapy = { version = "^2.11.0", optional = true }
 
-#dev = [
-#    "build ~= 1.0.3",
-#    "filelock ~= 3.12.4",
-#    "mypy ~= 1.7.1",
-#    "pre-commit ~= 3.4.0",
-#    "pydoc-markdown ~= 4.8.2",
-#    "pytest ~= 7.4.2",
-#    "pytest-asyncio ~= 0.21.0",
-#    "pytest-cov ~= 4.1.0",
-#    "pytest-only ~= 2.0.0",
-#    "pytest-timeout ~= 2.2.0",
-#    "pytest-xdist ~= 3.3.1",
-#    "respx ~= 0.20.1",
-#    "ruff ~= 0.1.13",
-#    "twine ~= 4.0.2",
-#    "types-aiofiles ~= 23.2.0.0",
-#    "types-colorama ~= 0.4.15.12",
-#    "types-psutil ~= 5.9.5.17",
-#    "scrapy >= 2.11.0"
-#]
+
+[tool.poetry.extras]
+scrapy = ["scrapy"]
 
 [tool.poetry.urls]
 "Homepage" = "https://docs.apify.com/sdk/python/"
@@ -108,31 +77,31 @@ build-backend = "poetry.core.masonry.api"
 line-length = 150
 select = ["ALL"]
 ignore = [
-    "ANN401",  # Dynamically typed expressions (typing.Any) are disallowed in {filename}
-    "BLE001",  # Do not catch blind exception
-    "C901",    # `{name}` is too complex
-    "COM812",  # This rule may cause conflicts when used with the formatter
-    "D100",    # Missing docstring in public module
-    "D104",    # Missing docstring in public package
-    "EM",      # flake8-errmsg
-    "G004",    # Logging statement uses f-string
-    "ISC001",  # This rule may cause conflicts when used with the formatter
-    "FIX",     # flake8-fixme
-    "PGH003",  # Use specific rule codes when ignoring type issues
+    "ANN401", # Dynamically typed expressions (typing.Any) are disallowed in {filename}
+    "BLE001", # Do not catch blind exception
+    "C901", # `{name}` is too complex
+    "COM812", # This rule may cause conflicts when used with the formatter
+    "D100", # Missing docstring in public module
+    "D104", # Missing docstring in public package
+    "EM", # flake8-errmsg
+    "G004", # Logging statement uses f-string
+    "ISC001", # This rule may cause conflicts when used with the formatter
+    "FIX", # flake8-fixme
+    "PGH003", # Use specific rule codes when ignoring type issues
     "PLR0911", # Too many return statements
     "PLR0913", # Too many arguments in function definition
     "PLR0915", # Too many statements
-    "PTH",     # flake8-use-pathlib
-    "PYI034",  # `__aenter__` methods in classes like `{name}` usually return `self` at runtime
-    "PYI036",  # The second argument in `__aexit__` should be annotated with `object` or `BaseException | None`
-    "S102",    # Use of `exec` detected
-    "S105",    # Possible hardcoded password assigned to
-    "S106",    # Possible hardcoded password assigned to argument: "{name}"
-    "S301",    # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue
-    "S303",    # Use of insecure MD2, MD4, MD5, or SHA1 hash function
-    "S311",    # Standard pseudo-random generators are not suitable for cryptographic purposes
-    "TD002",   # Missing author in TODO; try: `# TODO(<author_name>): ...` or `# TODO @<author_name>: ...
-    "TRY003",  # Avoid specifying long messages outside the exception class
+    "PTH", # flake8-use-pathlib
+    "PYI034", # `__aenter__` methods in classes like `{name}` usually return `self` at runtime
+    "PYI036", # The second argument in `__aexit__` should be annotated with `object` or `BaseException | None`
+    "S102", # Use of `exec` detected
+    "S105", # Possible hardcoded password assigned to
+    "S106", # Possible hardcoded password assigned to argument: "{name}"
+    "S301", # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue
+    "S303", # Use of insecure MD2, MD4, MD5, or SHA1 hash function
+    "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes
+    "TD002", # Missing author in TODO; try: `# TODO(<author_name>): ...` or `# TODO @<author_name>: ...
+    "TRY003", # Avoid specifying long messages outside the exception class
 
     # TODO: Remove this once the following issue is fixed
     # https://github.com/apify/apify-sdk-python/issues/150
@@ -148,20 +117,20 @@ indent-style = "space"
     "F401", # Unused imports
 ]
 "**/{scripts}/*" = [
-    "D",       # Everything from the pydocstyle
-    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
+    "D", # Everything from the pydocstyle
+    "INP001", # File {filename} is part of an implicit namespace package, add an __init__.py
     "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable
-    "T20",     # flake8-print
+    "T20", # flake8-print
 ]
 "**/{tests}/*" = [
-    "D",       # Everything from the pydocstyle
-    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
-    "PT011",   # `pytest.raises({ExceptionType})` is too broad, set the `match` parameter or use a more specific exception
+    "D", # Everything from the pydocstyle
+    "INP001", # File {filename} is part of an implicit namespace package, add an __init__.py
+    "PT011", # `pytest.raises({ExceptionType})` is too broad, set the `match` parameter or use a more specific exception
     "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable
-    "S101",    # Use of assert detected
-    "T20",     # flake8-print
-    "TID252",  # Relative imports from parent modules are banned
-    "TRY301",  # Abstract `raise` to an inner function
+    "S101", # Use of assert detected
+    "T20", # flake8-print
+    "TID252", # Relative imports from parent modules are banned
+    "TRY301", # Abstract `raise` to an inner function
 ]
 
 [tool.ruff.lint.flake8-quotes]

From 9f172ffc1f0300ffa649e58755d60b7eb53d6c5b Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Fri, 16 Feb 2024 20:05:18 +0200
Subject: [PATCH 12/18] second attempt create extra requirement scrapy.

Signed-off-by: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
---
 pyproject.toml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 581bb84a..dc7278b2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -53,12 +53,9 @@ twine = "^4.0.2"
 types-aiofiles = "^23.2.0.0"
 types-colorama = "^0.4.15.12"
 types-psutil = "^5.9.5.17"
-scrapy = { version = "^2.11.0", optional = true }
+scrapy = { version = "^2.11.0", optional = true, extras = ["scrapy"] }
 
 
-[tool.poetry.extras]
-scrapy = ["scrapy"]
-
 [tool.poetry.urls]
 "Homepage" = "https://docs.apify.com/sdk/python/"
 "Documentation" = "https://docs.apify.com/sdk/python/"

From 6a738920cf7a77e4479d5c8ff65a2b4be3e439d7 Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Fri, 16 Feb 2024 20:12:41 +0200
Subject: [PATCH 13/18] include extra dependencies for poetry install.

Signed-off-by: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 4a99aef9..f4421c05 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,7 @@ install-dev:
 	pip install poetry
 	poetry config virtualenvs.create true --local
 	poetry config virtualenvs.in-project true --local
-	poetry install
+	poetry install --all-extras
 	poetry run pre-commit install
 
 build:

From d2b6f29eac61af934f27a2b20068df321b9cd3cc Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Fri, 16 Feb 2024 20:17:00 +0200
Subject: [PATCH 14/18] include extra dependencies for poetry install.

Signed-off-by: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
---
 pyproject.toml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pyproject.toml b/pyproject.toml
index dc7278b2..3db9d6e8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -55,6 +55,8 @@ types-colorama = "^0.4.15.12"
 types-psutil = "^5.9.5.17"
 scrapy = { version = "^2.11.0", optional = true, extras = ["scrapy"] }
 
+[tool.poetry.extras]
+scrapy = ["scrapy"]
 
 [tool.poetry.urls]
 "Homepage" = "https://docs.apify.com/sdk/python/"

From 1795f18f40f1075c4d5243152acc6ef0e7e9def2 Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Fri, 16 Feb 2024 20:28:29 +0200
Subject: [PATCH 15/18] include extra dependencies for poetry install.

Signed-off-by: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 3db9d6e8..ef7539c9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -53,7 +53,7 @@ twine = "^4.0.2"
 types-aiofiles = "^23.2.0.0"
 types-colorama = "^0.4.15.12"
 types-psutil = "^5.9.5.17"
-scrapy = { version = "^2.11.0", optional = true, extras = ["scrapy"] }
+scrapy = { version = "^2.11.0",  extras = ["scrapy"] }
 
 [tool.poetry.extras]
 scrapy = ["scrapy"]

From 8054c176ab57ffe2c385dea6ac899a97da9670b6 Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Fri, 16 Feb 2024 21:09:56 +0200
Subject: [PATCH 16/18] updated unittests ci with poetry implementation.

Signed-off-by: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
---
 .github/workflows/run_checks.yaml       |  5 ----
 .github/workflows/unit_tests.yaml       | 25 +++++++++++-----
 .github/workflows/unit_tests_poetry.yml | 38 -------------------------
 3 files changed, 18 insertions(+), 50 deletions(-)
 delete mode 100644 .github/workflows/unit_tests_poetry.yml

diff --git a/.github/workflows/run_checks.yaml b/.github/workflows/run_checks.yaml
index 9128f2eb..93dcff3b 100644
--- a/.github/workflows/run_checks.yaml
+++ b/.github/workflows/run_checks.yaml
@@ -17,11 +17,6 @@ jobs:
     needs: [lint_and_type_checks]
     uses: ./.github/workflows/unit_tests.yaml
 
-  unit_tests_poetry:
-    name: Run unit tests
-    needs: [ lint_and_type_checks ]
-    uses: ./.github/workflows/unit_tests_poetry.yml
-
   # If the PR comes from the main repo, run integration tests directly
   integration_tests:
     if: github.event.pull_request.head.repo.owner.login == 'apify'
diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml
index a5e3691d..fd24be15 100644
--- a/.github/workflows/unit_tests.yaml
+++ b/.github/workflows/unit_tests.yaml
@@ -8,8 +8,8 @@ jobs:
     name: Run unit tests
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest]
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -25,8 +25,19 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install dependencies
-        run: make install-dev
-
-      - name: Run unit tests
-        run: make unit-tests
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v2
+      - name: Setup a local virtual environment (if no poetry.toml file)
+        run: |
+          poetry config virtualenvs.create true --local
+          poetry config virtualenvs.in-project true --local
+      - uses: actions/cache@v3
+        name: Define a cache for the virtual environment based on the dependencies lock file
+        with:
+          path: ./.venv
+          key: venv-${{ hashFiles('poetry.lock') }}
+      - name: Install the project dependencies
+        run: poetry install
+      - name: Run the automated tests (for example)
+        run: poetry run pytest tests/unit
+  continue-on-error: true
diff --git a/.github/workflows/unit_tests_poetry.yml b/.github/workflows/unit_tests_poetry.yml
deleted file mode 100644
index da4700fe..00000000
--- a/.github/workflows/unit_tests_poetry.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-name: Unit tests
-
-on:
-  [ pull_request, push]
-
-jobs:
-    unit-test:
-        runs-on: ${{ matrix.os }}
-        strategy:
-            matrix:
-                os: [ ubuntu-latest, macos-latest, windows-latest ]
-                python-version: [ '3.9', '3.10', '3.11', '3.12' ]
-
-        steps:
-            -   uses: actions/checkout@v3
-
-            -   name: Install Python
-                uses: actions/setup-python@v4
-                with:
-                    python-version: ${{ matrix.python-version }}
-
-            -   name: Install poetry
-                uses: abatilo/actions-poetry@v2
-            -   name: Setup a local virtual environment (if no poetry.toml file)
-                run: |
-                    poetry config virtualenvs.create true --local
-                    poetry config virtualenvs.in-project true --local
-            -   uses: actions/cache@v3
-                name: Define a cache for the virtual environment based on the dependencies lock file
-                with:
-                    path: ./.venv
-                    key: venv-${{ hashFiles('poetry.lock') }}
-            -   name: Install the project dependencies
-                run: poetry install
-            -   name: Run the automated tests (for example)
-                run: poetry run pytest tests/unit
-        continue-on-error: true
-

From 2589fd2e1904510d6ced8ee76e948970d0ca33e7 Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Sat, 17 Feb 2024 15:39:25 +0200
Subject: [PATCH 17/18] schema fix.

Signed-off-by: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
---
 .github/workflows/unit_tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml
index fd24be15..826d85d5 100644
--- a/.github/workflows/unit_tests.yaml
+++ b/.github/workflows/unit_tests.yaml
@@ -40,4 +40,4 @@ jobs:
         run: poetry install
       - name: Run the automated tests (for example)
         run: poetry run pytest tests/unit
-  continue-on-error: true
+    continue-on-error: true

From f0bbc86b58ea94a1eb46cab75b34c43d51fd6eaf Mon Sep 17 00:00:00 2001
From: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
Date: Tue, 20 Feb 2024 10:08:37 +0200
Subject: [PATCH 18/18] fix build with poetry.

Signed-off-by: DanielAvdar <66269169+DanielAvdar@users.noreply.github.com>
---
 Makefile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index f4421c05..c92fec7f 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,6 @@ clean:
 	rm -rf build dist .mypy_cache .pytest_cache src/*.egg-info __pycache__
 
 install-dev:
-
 	pip install poetry
 	poetry config virtualenvs.create true --local
 	poetry config virtualenvs.in-project true --local
@@ -17,7 +16,7 @@ install-dev:
 	poetry run pre-commit install
 
 build:
-	poetry run build
+	poetry run python -m build
 
 publish:
 	poetry run twine upload dist/*