Sync project setup with Python SDK (#119)

Author: fnesveda

.flake8

@@ -5,7 +5,11 @@ filename =
     ./src/*.py,
     ./tests/*.py,
     ./setup.py
-max_line_length = 150
+per-file-ignores =
+    docs/*: D
+    scripts/*: D
+    tests/*: D
+
 # Google docstring convention + D204 & D401
 docstring-convention = all
 ignore =
@@ -20,9 +24,9 @@ ignore =
     D409
     D413
     U101
+
+max_line_length = 150
 unused-arguments-ignore-overload-functions = True
 unused-arguments-ignore-stub-functions = True
-per-file-ignores =
-    docs/*: D
-    scripts/*: D
-    tests/*: D
+pytest-fixture-no-parentheses = True
+pytest-mark-no-parentheses = True

.github/workflows/integration_tests.yaml

@@ -0,0 +1,37 @@
+name: Integration tests
+
+on:
+  workflow_call:
+    secrets:
+      APIFY_TEST_USER_API_TOKEN:
+        description: API token of the Python SDK testing user on Apify
+        required: true
+
+concurrency: # This is to make sure that only one run of this workflow is running at the same time, to not overshoot the test user limits
+  group: integration_tests
+
+jobs:
+  integration_tests:
+    name: Run integration tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+      max-parallel: 1 # no concurrency on this level, to not overshoot the test user limits
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: make install-dev
+
+    - name: Run integration tests
+      run: make INTEGRATION_TESTS_CONCURRENCY=8 integration-tests
+      env:
+        APIFY_TEST_USER_API_TOKEN: ${{ secrets.APIFY_TEST_USER_API_TOKEN }}

.github/workflows/release.yaml

@@ -32,13 +32,18 @@ jobs:
     name: Run unit tests
     uses: ./.github/workflows/unit_tests.yaml
 
+  integration_tests:
+    name: Run integration tests
+    uses: ./.github/workflows/integration_tests.yaml
+    secrets: inherit
+
   check_docs:
     name: Check whether the documentation is up to date
     uses: ./.github/workflows/check_docs.yaml
 
   deploy:
     name: Publish to PyPI
-    needs: [lint_and_type_checks, unit_tests, check_docs]
+    needs: [lint_and_type_checks, unit_tests, integration_tests, check_docs]
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/run_checks.yaml

@@ -17,3 +17,8 @@ jobs:
     name: Check whether the documentation is up to date
     uses: ./.github/workflows/check_docs.yaml
 
+  integration_tests:
+    name: Run integration tests
+    needs: [lint_and_type_checks, unit_tests, check_docs]
+    uses: ./.github/workflows/integration_tests.yaml
+    secrets: inherit

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
       language: system
       pass_filenames: false
 
-    - id: unit-test
+    - id: unit-tests
       name: "Run unit tests"
       entry: "make unit-tests"
       language: system

Makefile

@@ -1,4 +1,7 @@
-.PHONY: clean install-dev lint unit-tests type-check check-code format docs check-docs check-async-docstrings fix-async-docstrings check-changelog-entry
+.PHONY: clean install-dev lint unit-tests integration-tests type-check check-code format docs check-docs check-async-docstrings fix-async-docstrings check-changelog-entry
+
+# This is default for local testing, but GitHub workflows override it to a higher value in CI
+INTEGRATION_TESTS_CONCURRENCY = 1
 
 clean:
 	rm -rf build dist .mypy_cache .pytest_cache src/*.egg-info __pycache__
@@ -15,6 +18,9 @@ lint:
 unit-tests:
 	python3 -m pytest -n auto -ra tests/unit
 
+integration-tests:
+	python3 -m pytest -n $(INTEGRATION_TESTS_CONCURRENCY) -ra tests/integration
+
 type-check:
 	python3 -m mypy
 

setup.py

@@ -53,15 +53,21 @@
     packages=find_packages(where='src'),
     package_data={'apify_client': ['py.typed']},
     python_requires='>=3.8',
-    install_requires=['httpx ~= 0.23.0'],
+    install_requires=[
+        'httpx ~= 0.23.0',
+    ],
     extras_require={
         'dev': [
             'autopep8 ~= 2.0.1',
             'flake8 ~= 6.0.0',
             'flake8-bugbear ~= 23.1.20',
             'flake8-commas ~= 2.1.0',
+            'flake8-comprehensions ~= 3.10.1',
+            'flake8-datetimez ~= 20.10.0',
             'flake8-docstrings ~= 1.7.0',
             'flake8-isort ~= 6.0.0',
+            'flake8-noqa ~= 1.3.0',
+            'flake8-pytest-style ~= 1.7.2',
             'flake8-quotes ~= 3.3.1',
             'flake8-unused-arguments ~= 0.0.13',
             'isort ~= 5.12.0',
@@ -70,6 +76,7 @@
             'pre-commit ~= 3.0.1',
             'pytest ~= 7.2.0',
             'pytest-asyncio ~= 0.20.3',
+            'pytest-only ~= 2.0.0',
             'pytest-randomly ~= 3.12.0',
             'pytest-timeout ~= 2.1.0',
             'pytest-xdist ~= 3.2.0',

src/apify_client/clients/base/actor_job_base_client.py

@@ -1,7 +1,7 @@
 import asyncio
 import math
 import time
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Dict, Optional
 
 from ..._errors import ApifyApiError
@@ -20,7 +20,7 @@ class ActorJobBaseClient(ResourceClient):
     """Base sub-client class for actor runs and actor builds."""
 
     def _wait_for_finish(self, wait_secs: Optional[int] = None) -> Optional[Dict]:
-        started_at = datetime.now()
+        started_at = datetime.now(timezone.utc)
         should_repeat = True
         job: Optional[Dict] = None
         seconds_elapsed = 0
@@ -38,7 +38,7 @@ def _wait_for_finish(self, wait_secs: Optional[int] = None) -> Optional[Dict]:
                 )
                 job = _parse_date_fields(_pluck_data(response.json()))
 
-                seconds_elapsed = math.floor(((datetime.now() - started_at).total_seconds()))
+                seconds_elapsed = math.floor(((datetime.now(timezone.utc) - started_at).total_seconds()))
                 if (
                     ActorJobStatus(job['status'])._is_terminal or (wait_secs is not None and seconds_elapsed >= wait_secs)
                 ):
@@ -77,7 +77,7 @@ class ActorJobBaseClientAsync(ResourceClientAsync):
     """Base async sub-client class for actor runs and actor builds."""
 
     async def _wait_for_finish(self, wait_secs: Optional[int] = None) -> Optional[Dict]:
-        started_at = datetime.now()
+        started_at = datetime.now(timezone.utc)
         should_repeat = True
         job: Optional[Dict] = None
         seconds_elapsed = 0
@@ -95,7 +95,7 @@ async def _wait_for_finish(self, wait_secs: Optional[int] = None) -> Optional[Di
                 )
                 job = _parse_date_fields(_pluck_data(response.json()))
 
-                seconds_elapsed = math.floor(((datetime.now() - started_at).total_seconds()))
+                seconds_elapsed = math.floor(((datetime.now(timezone.utc) - started_at).total_seconds()))
                 if (
                     ActorJobStatus(job['status'])._is_terminal or (wait_secs is not None and seconds_elapsed >= wait_secs)
                 ):

tests/integration/conftest.py

@@ -0,0 +1,35 @@
+import os
+
+import pytest
+
+from apify_client import ApifyClient, ApifyClientAsync
+
+TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN'
+API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL'
+
+
+@pytest.fixture
+def apify_client() -> ApifyClient:
+    api_token = os.getenv(TOKEN_ENV_VAR)
+    api_url = os.getenv(API_URL_ENV_VAR)
+
+    if not api_token:
+        raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!')
+
+    return ApifyClient(api_token, api_url=api_url)
+
+
+# This fixture can't be session-scoped,
+# because then you start getting `RuntimeError: Event loop is closed` errors,
+# because `httpx.AsyncClient` in `ApifyClientAsync` tries to reuse the same event loop across requests,
+# but `pytest-asyncio` closes the event loop after each test,
+# and uses a new one for the next test.
+@pytest.fixture
+def apify_client_async() -> ApifyClientAsync:
+    api_token = os.getenv(TOKEN_ENV_VAR)
+    api_url = os.getenv(API_URL_ENV_VAR)
+
+    if not api_token:
+        raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!')
+
+    return ApifyClientAsync(api_token, api_url=api_url)

tests/integration/test_basic.py

@@ -0,0 +1,17 @@
+from apify_client import ApifyClient, ApifyClientAsync
+
+
+class TestBasicSync:
+    def test_basic(self, apify_client: ApifyClient) -> None:
+        me = apify_client.user('me').get()
+        assert me is not None
+        assert me.get('id') is not None
+        assert me.get('username') is not None
+
+
+class TestBasicAsync:
+    async def test_basic(self, apify_client_async: ApifyClientAsync) -> None:
+        me = await apify_client_async.user('me').get()
+        assert me is not None
+        assert me.get('id') is not None
+        assert me.get('username') is not None