Merge pull request #221 from reportportal/develop

Release

Author: HardNorth

.github/workflows/tests.yml

@@ -50,10 +50,11 @@ jobs:
           pip install tox tox-gh-actions
 
       - name: Test with tox
+        timeout-minutes: 10
         run: tox
 
       - name: Upload coverage to Codecov
-        if: matrix.python-version == 3.7 && success()
+        if: matrix.python-version == 3.8 && success()
         uses: codecov/codecov-action@v3
         with:
           files: coverage.xml

CHANGELOG.md

@@ -1,6 +1,18 @@
 # Changelog
 
 ## [Unreleased]
+### Added
+- `RP` class in `reportportal_client.client` module as common interface for all ReportPortal clients, by @HardNorth
+- `reportportal_client.aio` with asynchronous clients and auxiliary classes, by @HardNorth
+- Dependency on `aiohttp` and `certifi`, by @HardNorth
+### Changed
+- RPClient class does not use separate Thread for log processing anymore, by @HardNorth
+- Use `importlib.metadata` package for distribution data extraction for Python versions starting 3.8, by @HardNorth
+- `helpers.verify_value_length` function updated to truncate attribute keys also and reveal attributes were truncated, by @HardNorth
+### Removed
+- Dependency on `six`, by @HardNorth
+
+## [5.4.1]
 ### Changed
 - Unified ReportPortal product naming, by @HardNorth
 - `RPClient` internal item stack implementation changed to `LifoQueue` to maintain concurrency better, by @HardNorth

MANIFEST.in

@@ -1,2 +1,3 @@
 include MANIFEST.in
 include README.md CONTRIBUTING.rst requirements.txt
+exclude test_res/*

reportportal_client/__init__.py

@@ -1,28 +1,124 @@
-"""
-Copyright (c) 2022 https://reportportal.io .
+#  Copyright (c) 2023 EPAM Systems
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+"""This package is the base package for ReportPortal client."""
+import typing
+import warnings
 
-https://www.apache.org/licenses/LICENSE-2.0
+import aenum
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
+# noinspection PyProtectedMember
+from reportportal_client._internal.local import current, set_current
+from reportportal_client.aio.client import AsyncRPClient, BatchedRPClient, ThreadedRPClient
+from reportportal_client.client import RP, RPClient, OutputType
+from reportportal_client.logs import RPLogger, RPLogHandler
+from reportportal_client.steps import step
+
+
+class ClientType(aenum.Enum):
+    """Enum of possible type of ReportPortal clients."""
+
+    SYNC = aenum.auto()
+    ASYNC = aenum.auto()
+    ASYNC_THREAD = aenum.auto()
+    ASYNC_BATCHED = aenum.auto()
+
+
+# noinspection PyIncorrectDocstring
+def create_client(
+        client_type: ClientType,
+        endpoint: str,
+        project: str,
+        *,
+        api_key: str = None,
+        **kwargs: typing.Any
+) -> typing.Optional[RP]:
+    """Create and ReportPortal Client based on the type and arguments provided.
+
+    :param client_type:             Type of the Client to create.
+    :type client_type:              ClientType
+    :param endpoint:                Endpoint of the ReportPortal service.
+    :type endpoint:                 str
+    :param project:                 Project name to report to.
+    :type project:                  str
+    :param api_key:                 Authorization API key.
+    :type api_key:                  str
+    :param launch_uuid:             A launch UUID to use instead of starting own one.
+    :type launch_uuid:              str
+    :param is_skipped_an_issue:     Option to mark skipped tests as not 'To Investigate' items on the server
+                                    side.
+    :type is_skipped_an_issue:      bool
+    :param verify_ssl:              Option to skip ssl verification.
+    :type verify_ssl:               typing.Union[bool, str]
+    :param retries:                 Number of retry attempts to make in case of connection / server
+                                    errors.
+    :type retries:                  int
+    :param max_pool_size:           Option to set the maximum number of connections to save the pool.
+    :type max_pool_size:            int
+    :param http_timeout :           A float in seconds for connect and read timeout. Use a Tuple to
+                                    specific connect and read separately.
+    :type http_timeout:             Tuple[float, float]
+    :param mode:                    Launch mode, all Launches started by the client will be in that mode.
+    :type mode:                     str
+    :param launch_uuid_print:       Print Launch UUID into passed TextIO or by default to stdout.
+    :type launch_uuid_print:        bool
+    :param print_output:            Set output stream for Launch UUID printing.
+    :type print_output:             OutputType
+    :param log_batch_size:          Option to set the maximum number of logs that can be processed in one
+                                    batch.
+    :type log_batch_size:           int
+    :param log_batch_payload_limit: Maximum size in bytes of logs that can be processed in one batch.
+    :type log_batch_payload_limit:  int
+    :param keepalive_timeout:       For Async Clients only. Maximum amount of idle time in seconds before
+                                    force connection closing.
+    :type keepalive_timeout:        int
+    :param task_timeout:            For Async Threaded and Batched Clients only. Time limit in seconds for a
+                                    Task processing.
+    :type task_timeout:             float
+    :param shutdown_timeout:        For Async Threaded and Batched Clients only. Time limit in seconds for
+                                    shutting down internal Tasks.
+    :type shutdown_timeout:         float
+    :param trigger_num:             For Async Batched Client only. Number of tasks which triggers Task batch
+                                    execution.
+    :type trigger_num:              int
+    :param trigger_interval:        For Async Batched Client only. Time limit which triggers Task batch
+                                    execution.
+    :type trigger_interval:         float
+    :return: ReportPortal Client instance.
+    """
+    if client_type is ClientType.SYNC:
+        return RPClient(endpoint, project, api_key=api_key, **kwargs)
+    if client_type is ClientType.ASYNC:
+        return AsyncRPClient(endpoint, project, api_key=api_key, **kwargs)
+    if client_type is ClientType.ASYNC_THREAD:
+        return ThreadedRPClient(endpoint, project, api_key=api_key, **kwargs)
+    if client_type is ClientType.ASYNC_BATCHED:
+        return BatchedRPClient(endpoint, project, api_key=api_key, **kwargs)
+    warnings.warn(f'Unknown ReportPortal Client type requested: {client_type}', RuntimeWarning, stacklevel=2)
 
-from ._local import current
-from .logs import RPLogger, RPLogHandler
-from .client import RPClient
-from .steps import step
 
 __all__ = [
+    'ClientType',
+    'create_client',
     'current',
+    'set_current',
+    'RP',
+    'RPClient',
+    'AsyncRPClient',
+    'BatchedRPClient',
+    'ThreadedRPClient',
+    'OutputType',
     'RPLogger',
     'RPLogHandler',
-    'RPClient',
     'step',
 ]

reportportal_client/_internal/__init__.py

@@ -0,0 +1,14 @@
+#  Copyright (c) 2023 EPAM Systems
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License
+
+"""ReportPortal client internal API. No warnings before breaking changes. No backward compatibility."""

tests/logs/__init__.py renamed to reportportal_client/_internal/aio/__init__.py

@@ -1,6 +1,4 @@
-"""This package contains unit tests for logging."""
-
-#  Copyright (c) 2022 EPAM Systems
+#  Copyright (c) 2023 EPAM Systems
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at

reportportal_client/_internal/aio/http.py

@@ -0,0 +1,164 @@
+#  Copyright (c) 2023 EPAM Systems
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License
+
+"""This module designed to help with asynchronous HTTP request/response handling."""
+
+import asyncio
+import sys
+from types import TracebackType
+from typing import Coroutine, Any, Optional, Type, Callable
+
+from aenum import Enum
+from aiohttp import ClientSession, ClientResponse, ServerConnectionError, \
+    ClientResponseError
+
+DEFAULT_RETRY_NUMBER: int = 5
+DEFAULT_RETRY_DELAY: float = 0.005
+THROTTLING_STATUSES: set = {425, 429}
+RETRY_STATUSES: set = {408, 500, 502, 503, 504, 507}.union(THROTTLING_STATUSES)
+
+
+class RetryClass(int, Enum):
+    """Enum contains error types and their retry delay multiply factor as values."""
+
+    SERVER_ERROR = 1
+    CONNECTION_ERROR = 2
+    THROTTLING = 3
+
+
+class RetryingClientSession:
+    """Class uses aiohttp.ClientSession.request method and adds request retry logic."""
+
+    _client: ClientSession
+    __retry_number: int
+    __retry_delay: float
+
+    def __init__(
+            self,
+            *args,
+            max_retry_number: int = DEFAULT_RETRY_NUMBER,
+            base_retry_delay: float = DEFAULT_RETRY_DELAY,
+            **kwargs
+    ):
+        """Initialize an instance of the session with arguments.
+
+        To obtain the full list of arguments please see aiohttp.ClientSession.__init__() method. This class
+        just bypass everything to the base method, except two local arguments 'max_retry_number' and
+        'base_retry_delay'.
+
+        :param max_retry_number: the maximum number of the request retries if it was unsuccessful
+        :param base_retry_delay: base value for retry delay, determine how much time the class will wait after
+                                 an error. Real value highly depends on Retry Class and Retry attempt number,
+                                 since retries are performed in exponential delay manner
+        """
+        self._client = ClientSession(*args, **kwargs)
+        self.__retry_number = max_retry_number
+        self.__retry_delay = base_retry_delay
+
+    async def __nothing(self):
+        pass
+
+    def __sleep(self, retry_num: int, retry_factor: int) -> Coroutine:
+        if retry_num > 0:  # don't wait at the first retry attempt
+            delay = (((retry_factor * self.__retry_delay) * 1000) ** retry_num) / 1000
+            return asyncio.sleep(delay)
+        else:
+            return self.__nothing()
+
+    async def __request(
+            self, method: Callable, url, **kwargs: Any
+    ) -> ClientResponse:
+        """Make a request and retry if necessary.
+
+        The method retries requests depending on error class and retry number. For no-retry errors, such as
+        400 Bad Request it just returns result, for cases where it's reasonable to retry it does it in
+        exponential manner.
+        """
+        result = None
+        exceptions = []
+        for i in range(self.__retry_number + 1):  # add one for the first attempt, which is not a retry
+            retry_factor = None
+            try:
+                result = await method(url, **kwargs)
+            except Exception as exc:
+                exceptions.append(exc)
+                if isinstance(exc, ServerConnectionError) or isinstance(exc, ClientResponseError):
+                    retry_factor = RetryClass.CONNECTION_ERROR
+
+                if not retry_factor:
+                    raise exc
+
+            if result:
+                if result.ok or result.status not in RETRY_STATUSES:
+                    return result
+
+                if result.status in THROTTLING_STATUSES:
+                    retry_factor = RetryClass.THROTTLING
+                else:
+                    retry_factor = RetryClass.SERVER_ERROR
+
+            if i + 1 < self.__retry_number:
+                # don't wait at the last attempt
+                await self.__sleep(i, retry_factor)
+
+        if exceptions:
+            if len(exceptions) > 1:
+                if sys.version_info > (3, 10):
+                    # noinspection PyCompatibility
+                    raise ExceptionGroup(  # noqa: F821
+                        'During retry attempts the following exceptions happened',
+                        exceptions
+                    )
+                else:
+                    raise exceptions[-1]
+            else:
+                raise exceptions[0]
+        return result
+
+    def get(self, url: str, *, allow_redirects: bool = True,
+            **kwargs: Any) -> Coroutine[Any, Any, ClientResponse]:
+        """Perform HTTP GET request."""
+        return self.__request(self._client.get, url, allow_redirects=allow_redirects, **kwargs)
+
+    def post(self, url: str, *, data: Any = None, **kwargs: Any) -> Coroutine[Any, Any, ClientResponse]:
+        """Perform HTTP POST request."""
+        return self.__request(self._client.post, url, data=data, **kwargs)
+
+    def put(self, url: str, *, data: Any = None, **kwargs: Any) -> Coroutine[Any, Any, ClientResponse]:
+        """Perform HTTP PUT request."""
+        return self.__request(self._client.put, url, data=data, **kwargs)
+
+    def close(self) -> Coroutine:
+        """Gracefully close internal aiohttp.ClientSession class instance."""
+        return self._client.close()
+
+    async def __aenter__(self) -> "RetryingClientSession":
+        """Auxiliary method which controls what `async with` construction does on block enter."""
+        return self
+
+    async def __aexit__(
+            self,
+            exc_type: Optional[Type[BaseException]],
+            exc_val: Optional[BaseException],
+            exc_tb: Optional[TracebackType],
+    ) -> None:
+        """Auxiliary method which controls what `async with` construction does on block exit."""
+        await self.close()