Fix #157 Improve the stability of API mock servers in unit tests

Author: seratch

.github/workflows/ci-build.yml

@@ -9,12 +9,15 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 10
     strategy:
       matrix:
         python-version: ['3.6', '3.7', '3.8', '3.9']
-
+    env:
+      # default: multiprocessing
+      # threading is more stable on GitHub Actions
+      BOLT_PYTHON_MOCK_SERVER_MODE: threading
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -30,10 +33,41 @@ jobs:
       run: |
         pytest tests/slack_bolt/
         pytest tests/scenario_tests/
-    - name: Run tests for adapters
+    - name: Run tests for Socket Mode adapters
       run: |
         pip install -e ".[adapter]"
-        pytest tests/adapter_tests/
+        pytest tests/adapter_tests/socket_mode/
+    - name: Run tests for HTTP Mode adapters (AWS)
+      run: |
+        pytest tests/adapter_tests/aws/
+    - name: Run tests for HTTP Mode adapters (Bottle)
+      run: |
+        pytest tests/adapter_tests/bottle/
+    - name: Run tests for HTTP Mode adapters (CherryPy)
+      run: |
+        pytest tests/adapter_tests/cherrypy/
+    - name: Run tests for HTTP Mode adapters (Django)
+      run: |
+        pytest tests/adapter_tests/django/
+    - name: Run tests for HTTP Mode adapters (Falcon)
+      run: |
+        pytest tests/adapter_tests/falcon/
+    - name: Run tests for HTTP Mode adapters (Flask)
+      run: |
+        pytest tests/adapter_tests/flask/
+    - name: Run tests for HTTP Mode adapters (Pyramid)
+      run: |
+        pytest tests/adapter_tests/pyramid/
+    - name: Run tests for HTTP Mode adapters (Starlette)
+      run: |
+        pytest tests/adapter_tests/starlette/
+    - name: Run tests for HTTP Mode adapters (Tornado)
+      run: |
+        pytest tests/adapter_tests/tornado/
+    - name: Run tests for HTTP Mode adapters (asyncio-based libraries)
+      run: |
+        pip install -e ".[async]"
+        pytest tests/adapter_tests_async/
     - name: Run pytype verification (3.8 only)
       run: |
         python_version=`python -V`
@@ -42,12 +76,10 @@ jobs:
           pip install -e ".[adapter]"
           pip install "pytype" && pytype slack_bolt/
         fi
-    - name: Run all tests for codecov (3.6 only)
+    - name: Run all tests for codecov (3.9 only)
       run: |
         python_version=`python -V`
-        # TODO: As Python 3.6 works the most stably on GitHub Actions,
-        # we use the version for code coverage and async tests
-        if [ ${python_version:7:3} == "3.6" ]; then
+        if [ ${python_version:7:3} == "3.9" ]; then
           pip install -e ".[async]"
           pip install -e ".[testing]"
           pytest --cov=slack_bolt/ && bash <(curl -s https://codecov.io/bash)

tests/adapter_tests/aws/__init__.py

@@ -19,6 +19,7 @@
 from tests.mock_web_api_server import (
     setup_mock_web_api_server,
     cleanup_mock_web_api_server,
+    assert_auth_test_count,
 )
 from tests.utils import remove_os_env_temporarily, restore_os_env
 
@@ -110,7 +111,7 @@ def events() -> Response:
             headers=self.build_headers(timestamp, body),
         )
         assert response["statusCode"] == 200
-        assert self.mock_received_requests["/auth.test"] == 1
+        assert_auth_test_count(self, 1)
 
     def test_shortcuts(self):
         app = App(
@@ -157,7 +158,7 @@ def events() -> Response:
             headers=self.build_headers(timestamp, body),
         )
         assert response["statusCode"] == 200
-        assert self.mock_received_requests["/auth.test"] == 1
+        assert_auth_test_count(self, 1)
 
     def test_commands(self):
         app = App(
@@ -204,7 +205,7 @@ def events() -> Response:
             headers=self.build_headers(timestamp, body),
         )
         assert response["statusCode"] == 200
-        assert self.mock_received_requests["/auth.test"] == 1
+        assert_auth_test_count(self, 1)
 
     def test_lazy_listeners(self):
         app = App(
@@ -261,7 +262,7 @@ def say_it(say):
         )
         response: Response = slack_handler.handle(request)
         assert response.status_code == 200
-        assert self.mock_received_requests["/auth.test"] == 1
+        assert_auth_test_count(self, 1)
         assert self.mock_received_requests["/chat.postMessage"] == 1
 
     def test_oauth(self):

tests/adapter_tests/test_aws_chalice.py …ts/adapter_tests/aws/test_aws_chalice.pytests/adapter_tests/test_aws_chalice.py renamed to tests/adapter_tests/aws/test_aws_chalice.py tests/adapter_tests/test_aws_chalice.py renamed to tests/adapter_tests/aws/test_aws_chalice.py

@@ -14,6 +14,7 @@
 from tests.mock_web_api_server import (
     setup_mock_web_api_server,
     cleanup_mock_web_api_server,
+    assert_auth_test_count,
 )
 from tests.utils import remove_os_env_temporarily, restore_os_env
 
@@ -123,7 +124,7 @@ def event_handler():
         }
         response = SlackRequestHandler(app).handle(event, self.context)
         assert response["statusCode"] == 200
-        assert self.mock_received_requests["/auth.test"] == 1
+        assert_auth_test_count(self, 1)
 
         event = {
             "body": body,
@@ -134,7 +135,7 @@ def event_handler():
         }
         response = SlackRequestHandler(app).handle(event, self.context)
         assert response["statusCode"] == 200
-        assert self.mock_received_requests["/auth.test"] == 1
+        assert_auth_test_count(self, 1)
 
     @mock_lambda
     def test_shortcuts(self):
@@ -173,7 +174,7 @@ def shortcut_handler(ack):
         }
         response = SlackRequestHandler(app).handle(event, self.context)
         assert response["statusCode"] == 200
-        assert self.mock_received_requests["/auth.test"] == 1
+        assert_auth_test_count(self, 1)
 
         event = {
             "body": body,
@@ -184,7 +185,7 @@ def shortcut_handler(ack):
         }
         response = SlackRequestHandler(app).handle(event, self.context)
         assert response["statusCode"] == 200
-        assert self.mock_received_requests["/auth.test"] == 1
+        assert_auth_test_count(self, 1)
 
     @mock_lambda
     def test_commands(self):
@@ -223,7 +224,7 @@ def command_handler(ack):
         }
         response = SlackRequestHandler(app).handle(event, self.context)
         assert response["statusCode"] == 200
-        assert self.mock_received_requests["/auth.test"] == 1
+        assert_auth_test_count(self, 1)
 
         event = {
             "body": body,
@@ -234,7 +235,7 @@ def command_handler(ack):
         }
         response = SlackRequestHandler(app).handle(event, self.context)
         assert response["statusCode"] == 200
-        assert self.mock_received_requests["/auth.test"] == 1
+        assert_auth_test_count(self, 1)
 
     @mock_lambda
     def test_lazy_listeners(self):
@@ -279,7 +280,7 @@ def say_it(say):
         }
         response = SlackRequestHandler(app).handle(event, self.context)
         assert response["statusCode"] == 200
-        assert self.mock_received_requests["/auth.test"] == 1
+        assert_auth_test_count(self, 1)
         assert self.mock_received_requests["/chat.postMessage"] == 1
 
     @mock_lambda

tests/adapter_tests/test_aws_lambda.py …sts/adapter_tests/aws/test_aws_lambda.pytests/adapter_tests/test_aws_lambda.py renamed to tests/adapter_tests/aws/test_aws_lambda.py tests/adapter_tests/test_aws_lambda.py renamed to tests/adapter_tests/aws/test_aws_lambda.py

@@ -11,6 +11,7 @@
 from tests.mock_web_api_server import (
     setup_mock_web_api_server,
     cleanup_mock_web_api_server,
+    assert_auth_test_count,
 )
 from tests.utils import remove_os_env_temporarily, restore_os_env
 
@@ -109,7 +110,7 @@ def test_events(self):
             response_body = slack_events()
             assert response.status_code == 200
             assert response_body == ""
-            assert self.mock_received_requests["/auth.test"] == 1
+            assert_auth_test_count(self, 1)
 
     def test_shortcuts(self):
         input = {
@@ -138,7 +139,7 @@ def test_shortcuts(self):
             response_body = slack_events()
             assert response.status_code == 200
             assert response_body == ""
-            assert self.mock_received_requests["/auth.test"] == 1
+            assert_auth_test_count(self, 1)
 
     def test_commands(self):
         input = (
@@ -167,4 +168,4 @@ def test_commands(self):
             response_body = slack_events()
             assert response.status_code == 200
             assert response_body == ""
-            assert self.mock_received_requests["/auth.test"] == 1
+            assert_auth_test_count(self, 1)